{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "20492d5cb0fe0374",
   "metadata": {},
   "source": [
    "# 利用MLP进行自回归学习\n",
    "\n",
    "来自视频 [徒手实现循环神经网络--利用MLP进行自回归学习](https://www.bilibili.com/video/BV1mT421a7Hi)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f430a3768dfe084d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:11.937855Z",
     "start_time": "2025-08-23T14:41:10.296565Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'a': 0, 'b': 1, 'c': 2, 'd': 3, 'e': 4, 'f': 5, 'g': 6, 'h': 7, 'i': 8, 'j': 9, 'k': 10, 'l': 11, 'm': 12, 'n': 13, 'o': 14, 'p': 15, 'q': 16, 'r': 17, 's': 18, 't': 19, 'u': 20, 'v': 21, 'w': 22, 'x': 23, 'y': 24, 'z': 25}\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import string\n",
    "\n",
    "char2indx = {s: i for i, s in enumerate(string.ascii_lowercase)}\n",
    "print(char2indx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "66bb213ed8e9e1e2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:11.960953Z",
     "start_time": "2025-08-23T14:41:11.954919Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([11, 14, 21,  4])\n"
     ]
    }
   ],
   "source": [
    "example = 'love'\n",
    "idx = []\n",
    "\n",
    "for i in example:\n",
    "    idx.append(char2indx[i])\n",
    "\n",
    "idx = torch.tensor(idx)\n",
    "print(idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "155a52523fd02a55",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:12.141789Z",
     "start_time": "2025-08-23T14:41:12.136386Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 1., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0., 0.]]) torch.Size([4, 26])\n"
     ]
    }
   ],
   "source": [
    "# 使用独热编码，将文本转化为二维张量\n",
    "num_claz = len(char2indx.keys())\n",
    "x = F.one_hot(idx, num_classes=num_claz).float()  # 转化为独热编码，每个编码项含有 26 个项\n",
    "print(x, x.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8e6840ebaa177172",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:12.165119Z",
     "start_time": "2025-08-23T14:41:12.156532Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.3001e+00, -1.6818e+00, -1.5082e-01,  1.7082e+00, -9.2635e-01],\n",
      "        [ 4.8845e-01, -2.1747e-01,  1.4961e+00,  1.5945e+00, -5.7903e-01],\n",
      "        [-9.7222e-02,  1.3297e+00,  1.0115e+00,  6.9976e-01,  6.6921e-01],\n",
      "        [-3.8000e-01,  9.9111e-02,  9.1740e-01,  2.4051e+00, -7.3256e-01],\n",
      "        [-6.1487e-01, -8.7433e-01,  1.5873e+00, -5.6844e-02,  1.0736e-01],\n",
      "        [-1.0592e+00, -9.8994e-01, -8.1861e-01,  4.1408e-03,  1.7209e+00],\n",
      "        [-4.5777e-01, -4.3086e-02, -7.9654e-01, -6.8516e-01, -1.0535e+00],\n",
      "        [-1.0770e+00, -1.0838e+00,  5.7061e-01, -9.4706e-01, -1.6891e+00],\n",
      "        [ 3.8214e-01, -6.8782e-01,  1.4181e+00, -1.4422e+00,  7.5733e-01],\n",
      "        [ 3.9571e-01, -2.7872e-01, -2.1768e+00, -5.0323e-01, -1.5829e-02],\n",
      "        [-1.0955e-01,  8.6788e-01,  1.3023e-01,  5.4591e-01, -7.9197e-01],\n",
      "        [-4.1740e-01, -1.4668e+00,  9.4021e-01, -4.2164e-01,  9.9845e-01],\n",
      "        [-5.7634e-01, -9.7154e-01,  1.7017e-01,  7.8236e-01,  1.2114e+00],\n",
      "        [-1.1186e+00,  2.1148e-02, -4.7534e-01,  1.8132e+00,  4.4536e-03],\n",
      "        [-1.6101e-01,  9.5309e-02,  1.9679e-01, -1.5359e+00,  6.0348e-01],\n",
      "        [ 6.6759e-01,  1.7046e+00,  2.4058e+00,  1.3682e+00,  9.5542e-01],\n",
      "        [-1.5309e-01,  6.3520e-01, -4.1138e-01,  7.6759e-01, -1.7431e+00],\n",
      "        [ 9.5204e-01,  3.1083e-02,  1.4561e+00,  5.6628e-01, -1.1452e+00],\n",
      "        [-7.3506e-01, -1.4046e+00,  2.0233e-01,  9.9512e-01,  2.9495e-01],\n",
      "        [ 2.9764e-01,  3.9727e-02,  4.8907e-01,  1.3179e+00, -1.1209e+00],\n",
      "        [ 6.8047e-01, -2.1974e+00,  7.7197e-01, -1.3285e+00, -7.8462e-01],\n",
      "        [ 2.3939e-03,  1.3262e+00, -2.3948e+00, -3.8267e-01,  1.0917e+00],\n",
      "        [ 6.7129e-01, -1.0075e-02,  1.2609e+00, -2.1872e+00, -2.7779e-01],\n",
      "        [ 1.3894e+00,  7.7198e-01,  1.5649e+00, -1.3618e+00, -8.8185e-01],\n",
      "        [-1.5265e+00, -1.2704e+00, -3.9625e-01, -2.7062e-01, -2.2494e+00],\n",
      "        [-1.1071e+00, -5.2177e-01,  1.3266e+00, -3.7280e-01, -8.5091e-01]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(tensor([[-4.1740e-01, -1.4668e+00,  9.4021e-01, -4.2164e-01,  9.9845e-01],\n",
       "         [-1.6101e-01,  9.5309e-02,  1.9679e-01, -1.5359e+00,  6.0348e-01],\n",
       "         [ 2.3939e-03,  1.3262e+00, -2.3948e+00, -3.8267e-01,  1.0917e+00],\n",
       "         [-6.1487e-01, -8.7433e-01,  1.5873e+00, -5.6844e-02,  1.0736e-01]]),\n",
       " torch.Size([4, 5]))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dims = 5\n",
    "w = torch.randn(num_claz, dims)  # 随机生成一个 26 * 5 维的张量\n",
    "print(w)\n",
    "(x @ w), (x @ w).shape  # 相当于取出独热编码标志的那一行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dfe0d309bc395902",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:12.278078Z",
     "start_time": "2025-08-23T14:41:12.273554Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[-4.1740e-01, -1.4668e+00,  9.4021e-01, -4.2164e-01,  9.9845e-01],\n",
       "         [-1.6101e-01,  9.5309e-02,  1.9679e-01, -1.5359e+00,  6.0348e-01],\n",
       "         [ 2.3939e-03,  1.3262e+00, -2.3948e+00, -3.8267e-01,  1.0917e+00],\n",
       "         [-6.1487e-01, -8.7433e-01,  1.5873e+00, -5.6844e-02,  1.0736e-01]]),\n",
       " torch.Size([4, 5]))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 上述内容等价于\n",
    "w[idx], w[idx].shape  # 本质上就是从"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6f2028e9b7e0f64d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:12.344012Z",
     "start_time": "2025-08-23T14:41:12.339863Z"
    }
   },
   "outputs": [],
   "source": [
    "class Embedding:\n",
    "\n",
    "    def __init__(self, num_embeddings, embedding_dims):\n",
    "        self.weight = torch.randn(num_embeddings, embedding_dims)\n",
    "\n",
    "    def __call__(self, x):\n",
    "        self.out = self.weight[x]\n",
    "        return self.out\n",
    "\n",
    "    def parameters(self):\n",
    "        return [self.weight]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "50010cd0d11bdca7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:12.371843Z",
     "start_time": "2025-08-23T14:41:12.368339Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10, 5])\n"
     ]
    }
   ],
   "source": [
    "em = Embedding(num_claz, dims)\n",
    "x = torch.randint(0, num_claz, (10, ))\n",
    "print(em(x).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ae1119ec0fc085eb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:12.401916Z",
     "start_time": "2025-08-23T14:41:12.396911Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([20, 10, 5])\n"
     ]
    }
   ],
   "source": [
    "x = torch.randint(0, num_claz, (20, 10))  #  一共 20 个文本，每个文本都是 10 个长度\n",
    "print(em(x).shape)  # 输出的大小最后一个 5 是特征"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a8fe7bca4b7aeed",
   "metadata": {},
   "source": [
    "文本生成是不断增长的，所以会带来窗口大小的问题，所以解决方式之一就是限定窗口大小。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "571fec5f223e5838",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:12.477687Z",
     "start_time": "2025-08-23T14:41:12.439576Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9b019e5eb8afc402",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:12.491785Z",
     "start_time": "2025-08-23T14:41:12.485428Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_CudaDeviceProperties(name='NVIDIA GeForce RTX 4070 Laptop GPU', major=8, minor=9, total_memory=8187MB, multi_processor_count=36, uuid=32fb9e2a-f036-c0e5-406e-f17efd79171c, L2_cache_size=32MB)\n"
     ]
    }
   ],
   "source": [
    "for i in range(torch.cuda.device_count()):\n",
    "    print(torch.cuda.get_device_properties(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "400c3f39b4441d48",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.084368Z",
     "start_time": "2025-08-23T14:41:12.523668Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader\n",
    "from datasets import load_dataset\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "torch.manual_seed(12046)\n",
    "\n",
    "# https://huggingface.co/datasets/code-search-net/code_search_net/tree/main/data\n",
    "datasets = load_dataset('json', data_files='./datasets/python/final/jsonl/train/*.jsonl.gz')\n",
    "datasets = datasets['train'].filter(lambda x: 'apache/spark' in x['repo'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2af44537f1da00ef",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.096297Z",
     "start_time": "2025-08-23T14:41:15.092208Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def to_arrow_schema(schema):\n",
      "    \"\"\" Convert a schema from Spark to Arrow\n",
      "    \"\"\"\n",
      "    import pyarrow as pa\n",
      "    fields = [pa.field(field.name, to_arrow_type(field.dataType), nullable=field.nullable)\n",
      "              for field in schema]\n",
      "    return pa.schema(fields)\n"
     ]
    }
   ],
   "source": [
    "print(datasets[8]['original_string'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "26cc388eb3356824",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.137251Z",
     "start_time": "2025-08-23T14:41:15.132253Z"
    }
   },
   "outputs": [],
   "source": [
    "class CharTokenizer:\n",
    "\n",
    "    def __init__(self, data, begin_ind=0, end_ind=1):\n",
    "        # data: list[str]\n",
    "        chars = sorted(list(set(''.join(data))))\n",
    "        self.char2ind = {s: i + 2 for i, s in enumerate(chars)}  # 加 2 是预留特殊字符\n",
    "        self.char2ind['<|b|>'] = begin_ind\n",
    "        self.char2ind['<|e|>'] = end_ind\n",
    "        self.ind2char = {v: k for k, v in self.char2ind.items()}\n",
    "        self.begin_ind = begin_ind\n",
    "        self.end_ind = end_ind\n",
    "\n",
    "    def encode(self, x: str):\n",
    "        return [self.char2ind[i] for i in x]\n",
    "\n",
    "    def decode(self, x):\n",
    "        if isinstance(x, int):\n",
    "            return self.ind2char[x]\n",
    "        return [self.ind2char[i] for i in x]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8dc88f23deb7431d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.180981Z",
     "start_time": "2025-08-23T14:41:15.154153Z"
    }
   },
   "outputs": [],
   "source": [
    "tokenizer = CharTokenizer(datasets['original_string'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ac21148a3a3a2641",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.190054Z",
     "start_time": "2025-08-23T14:41:15.186510Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[71, 72, 73, 3, 73, 11, 91, 12, 29]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_str = 'def f(x):'\n",
    "tokens = tokenizer.encode(test_str)\n",
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cff2ccd80e712e9e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.231925Z",
     "start_time": "2025-08-23T14:41:15.226757Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'def f(x):'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join(tokenizer.decode(tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ae8a93745aed1f94",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.269447Z",
     "start_time": "2025-08-23T14:41:15.265941Z"
    }
   },
   "outputs": [],
   "source": [
    "# 这个函数主要用于将数据集处理为上下文窗口为 10 的文本\n",
    "def autoregressive_trans(text, tokenizer, context_length=10):\n",
    "    # text: str\n",
    "    inputs, labels = [], []\n",
    "    bind = tokenizer.begin_ind\n",
    "    eind = tokenizer.end_ind\n",
    "    enc = tokenizer.encode(text)\n",
    "\n",
    "    # 增加特殊字符\n",
    "    data = [bind] * context_length + enc + [eind]\n",
    "    for i in range(len(data) - context_length):\n",
    "        inputs.append(data[i: i + context_length])\n",
    "        labels.append(data[i + context_length])\n",
    "    return inputs, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b6d532a0db29fbb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.306397Z",
     "start_time": "2025-08-23T14:41:15.302348Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|b|><|b|><|b|> -----> d\n",
      "<|b|><|b|>d -----> e\n",
      "<|b|>de -----> f\n",
      "def ----->  \n",
      "ef  -----> f\n",
      "f f -----> (\n",
      " f( -----> x\n",
      "f(x -----> )\n",
      "(x) -----> :\n",
      "x): -----> <|e|>\n"
     ]
    }
   ],
   "source": [
    "inputs, labels = autoregressive_trans(test_str, tokenizer, 3)\n",
    "for a, b in zip(inputs, labels):\n",
    "    print(f'{\"\".join(tokenizer.decode(a))} -----> {tokenizer.decode(b)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a6675e9a0b4ffe71",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.329172Z",
     "start_time": "2025-08-23T14:41:15.325675Z"
    }
   },
   "outputs": [],
   "source": [
    "def process(data, tokenizer):\n",
    "    text = data['original_string']\n",
    "    # text: str\n",
    "    if isinstance(text, str):\n",
    "        inputs, labels = autoregressive_trans(text, tokenizer)\n",
    "        return {'inputs': inputs, 'labels': labels}\n",
    "    # text: list[str]\n",
    "    inputs, labels = [], []\n",
    "    for t in text:\n",
    "        i, l = autoregressive_trans(t, tokenizer)\n",
    "        inputs += i\n",
    "        labels += l\n",
    "    return {'inputs': inputs, 'labels': labels}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c95d56423f3cb94a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.375795Z",
     "start_time": "2025-08-23T14:41:15.355044Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'inputs': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n",
       "  [0, 0, 0, 0, 0, 0, 0, 0, 0, 71],\n",
       "  [0, 0, 0, 0, 0, 0, 0, 0, 71, 72],\n",
       "  [0, 0, 0, 0, 0, 0, 0, 71, 72, 73],\n",
       "  [0, 0, 0, 0, 0, 0, 71, 72, 73, 3],\n",
       "  [0, 0, 0, 0, 0, 71, 72, 73, 3, 87],\n",
       "  [0, 0, 0, 0, 71, 72, 73, 3, 87, 82],\n",
       "  [0, 0, 0, 71, 72, 73, 3, 87, 82, 66],\n",
       "  [0, 0, 71, 72, 73, 3, 87, 82, 66, 68],\n",
       "  [0, 71, 72, 73, 3, 87, 82, 66, 68, 85],\n",
       "  [71, 72, 73, 3, 87, 82, 66, 68, 85, 85],\n",
       "  [72, 73, 3, 87, 82, 66, 68, 85, 85, 82],\n",
       "  [73, 3, 87, 82, 66, 68, 85, 85, 82, 90],\n",
       "  [3, 87, 82, 66, 68, 85, 85, 82, 90, 66],\n",
       "  [87, 82, 66, 68, 85, 85, 82, 90, 66, 86],\n",
       "  [82, 66, 68, 85, 85, 82, 90, 66, 86, 70],\n",
       "  [66, 68, 85, 85, 82, 90, 66, 86, 70, 75],\n",
       "  [68, 85, 85, 82, 90, 66, 86, 70, 75, 72],\n",
       "  [85, 85, 82, 90, 66, 86, 70, 75, 72, 80],\n",
       "  [85, 82, 90, 66, 86, 70, 75, 72, 80, 68],\n",
       "  [82, 90, 66, 86, 70, 75, 72, 80, 68, 11],\n",
       "  [90, 66, 86, 70, 75, 72, 80, 68, 11, 86],\n",
       "  [66, 86, 70, 75, 72, 80, 68, 11, 86, 70],\n",
       "  [86, 70, 75, 72, 80, 68, 11, 86, 70, 75],\n",
       "  [70, 75, 72, 80, 68, 11, 86, 70, 75, 72],\n",
       "  [75, 72, 80, 68, 11, 86, 70, 75, 72, 80],\n",
       "  [72, 80, 68, 11, 86, 70, 75, 72, 80, 68],\n",
       "  [80, 68, 11, 86, 70, 75, 72, 80, 68, 12],\n",
       "  [68, 11, 86, 70, 75, 72, 80, 68, 12, 29],\n",
       "  [11, 86, 70, 75, 72, 80, 68, 12, 29, 2],\n",
       "  [86, 70, 75, 72, 80, 68, 12, 29, 2, 3],\n",
       "  [70, 75, 72, 80, 68, 12, 29, 2, 3, 3],\n",
       "  [75, 72, 80, 68, 12, 29, 2, 3, 3, 3],\n",
       "  [72, 80, 68, 12, 29, 2, 3, 3, 3, 3],\n",
       "  [80, 68, 12, 29, 2, 3, 3, 3, 3, 5],\n",
       "  [68, 12, 29, 2, 3, 3, 3, 3, 5, 5],\n",
       "  [12, 29, 2, 3, 3, 3, 3, 5, 5, 5],\n",
       "  [29, 2, 3, 3, 3, 3, 5, 5, 5, 3],\n",
       "  [2, 3, 3, 3, 3, 5, 5, 5, 3, 38],\n",
       "  [3, 3, 3, 3, 5, 5, 5, 3, 38, 82],\n",
       "  [3, 3, 3, 5, 5, 5, 3, 38, 82, 81],\n",
       "  [3, 3, 5, 5, 5, 3, 38, 82, 81, 89],\n",
       "  [3, 5, 5, 5, 3, 38, 82, 81, 89, 72],\n",
       "  [5, 5, 5, 3, 38, 82, 81, 89, 72, 85],\n",
       "  [5, 5, 3, 38, 82, 81, 89, 72, 85, 87],\n",
       "  [5, 3, 38, 82, 81, 89, 72, 85, 87, 3],\n",
       "  [3, 38, 82, 81, 89, 72, 85, 87, 3, 68],\n",
       "  [38, 82, 81, 89, 72, 85, 87, 3, 68, 3],\n",
       "  [82, 81, 89, 72, 85, 87, 3, 68, 3, 86],\n",
       "  [81, 89, 72, 85, 87, 3, 68, 3, 86, 70],\n",
       "  [89, 72, 85, 87, 3, 68, 3, 86, 70, 75],\n",
       "  [72, 85, 87, 3, 68, 3, 86, 70, 75, 72],\n",
       "  [85, 87, 3, 68, 3, 86, 70, 75, 72, 80],\n",
       "  [87, 3, 68, 3, 86, 70, 75, 72, 80, 68],\n",
       "  [3, 68, 3, 86, 70, 75, 72, 80, 68, 3],\n",
       "  [68, 3, 86, 70, 75, 72, 80, 68, 3, 73],\n",
       "  [3, 86, 70, 75, 72, 80, 68, 3, 73, 85],\n",
       "  [86, 70, 75, 72, 80, 68, 3, 73, 85, 82],\n",
       "  [70, 75, 72, 80, 68, 3, 73, 85, 82, 80],\n",
       "  [75, 72, 80, 68, 3, 73, 85, 82, 80, 3],\n",
       "  [72, 80, 68, 3, 73, 85, 82, 80, 3, 54],\n",
       "  [80, 68, 3, 73, 85, 82, 80, 3, 54, 83],\n",
       "  [68, 3, 73, 85, 82, 80, 3, 54, 83, 68],\n",
       "  [3, 73, 85, 82, 80, 3, 54, 83, 68, 85],\n",
       "  [73, 85, 82, 80, 3, 54, 83, 68, 85, 78],\n",
       "  [85, 82, 80, 3, 54, 83, 68, 85, 78, 3],\n",
       "  [82, 80, 3, 54, 83, 68, 85, 78, 3, 87],\n",
       "  [80, 3, 54, 83, 68, 85, 78, 3, 87, 82],\n",
       "  [3, 54, 83, 68, 85, 78, 3, 87, 82, 3],\n",
       "  [54, 83, 68, 85, 78, 3, 87, 82, 3, 36],\n",
       "  [83, 68, 85, 78, 3, 87, 82, 3, 36, 85],\n",
       "  [68, 85, 78, 3, 87, 82, 3, 36, 85, 85],\n",
       "  [85, 78, 3, 87, 82, 3, 36, 85, 85, 82],\n",
       "  [78, 3, 87, 82, 3, 36, 85, 85, 82, 90],\n",
       "  [3, 87, 82, 3, 36, 85, 85, 82, 90, 2],\n",
       "  [87, 82, 3, 36, 85, 85, 82, 90, 2, 3],\n",
       "  [82, 3, 36, 85, 85, 82, 90, 2, 3, 3],\n",
       "  [3, 36, 85, 85, 82, 90, 2, 3, 3, 3],\n",
       "  [36, 85, 85, 82, 90, 2, 3, 3, 3, 3],\n",
       "  [85, 85, 82, 90, 2, 3, 3, 3, 3, 5],\n",
       "  [85, 82, 90, 2, 3, 3, 3, 3, 5, 5],\n",
       "  [82, 90, 2, 3, 3, 3, 3, 5, 5, 5],\n",
       "  [90, 2, 3, 3, 3, 3, 5, 5, 5, 2],\n",
       "  [2, 3, 3, 3, 3, 5, 5, 5, 2, 3],\n",
       "  [3, 3, 3, 3, 5, 5, 5, 2, 3, 3],\n",
       "  [3, 3, 3, 5, 5, 5, 2, 3, 3, 3],\n",
       "  [3, 3, 5, 5, 5, 2, 3, 3, 3, 3],\n",
       "  [3, 5, 5, 5, 2, 3, 3, 3, 3, 76],\n",
       "  [5, 5, 5, 2, 3, 3, 3, 3, 76, 80],\n",
       "  [5, 5, 2, 3, 3, 3, 3, 76, 80, 83],\n",
       "  [5, 2, 3, 3, 3, 3, 76, 80, 83, 82],\n",
       "  [2, 3, 3, 3, 3, 76, 80, 83, 82, 85],\n",
       "  [3, 3, 3, 3, 76, 80, 83, 82, 85, 87],\n",
       "  [3, 3, 3, 76, 80, 83, 82, 85, 87, 3],\n",
       "  [3, 3, 76, 80, 83, 82, 85, 87, 3, 83],\n",
       "  [3, 76, 80, 83, 82, 85, 87, 3, 83, 92],\n",
       "  [76, 80, 83, 82, 85, 87, 3, 83, 92, 68],\n",
       "  [80, 83, 82, 85, 87, 3, 83, 92, 68, 85],\n",
       "  [83, 82, 85, 87, 3, 83, 92, 68, 85, 85],\n",
       "  [82, 85, 87, 3, 83, 92, 68, 85, 85, 82],\n",
       "  [85, 87, 3, 83, 92, 68, 85, 85, 82, 90],\n",
       "  [87, 3, 83, 92, 68, 85, 85, 82, 90, 3],\n",
       "  [3, 83, 92, 68, 85, 85, 82, 90, 3, 68],\n",
       "  [83, 92, 68, 85, 85, 82, 90, 3, 68, 86],\n",
       "  [92, 68, 85, 85, 82, 90, 3, 68, 86, 3],\n",
       "  [68, 85, 85, 82, 90, 3, 68, 86, 3, 83],\n",
       "  [85, 85, 82, 90, 3, 68, 86, 3, 83, 68],\n",
       "  [85, 82, 90, 3, 68, 86, 3, 83, 68, 2],\n",
       "  [82, 90, 3, 68, 86, 3, 83, 68, 2, 3],\n",
       "  [90, 3, 68, 86, 3, 83, 68, 2, 3, 3],\n",
       "  [3, 68, 86, 3, 83, 68, 2, 3, 3, 3],\n",
       "  [68, 86, 3, 83, 68, 2, 3, 3, 3, 3],\n",
       "  [86, 3, 83, 68, 2, 3, 3, 3, 3, 73],\n",
       "  [3, 83, 68, 2, 3, 3, 3, 3, 73, 76],\n",
       "  [83, 68, 2, 3, 3, 3, 3, 73, 76, 72],\n",
       "  [68, 2, 3, 3, 3, 3, 73, 76, 72, 79],\n",
       "  [2, 3, 3, 3, 3, 73, 76, 72, 79, 71],\n",
       "  [3, 3, 3, 3, 73, 76, 72, 79, 71, 86],\n",
       "  [3, 3, 3, 73, 76, 72, 79, 71, 86, 3],\n",
       "  [3, 3, 73, 76, 72, 79, 71, 86, 3, 32],\n",
       "  [3, 73, 76, 72, 79, 71, 86, 3, 32, 3],\n",
       "  [73, 76, 72, 79, 71, 86, 3, 32, 3, 62],\n",
       "  [76, 72, 79, 71, 86, 3, 32, 3, 62, 83],\n",
       "  [72, 79, 71, 86, 3, 32, 3, 62, 83, 68],\n",
       "  [79, 71, 86, 3, 32, 3, 62, 83, 68, 17],\n",
       "  [71, 86, 3, 32, 3, 62, 83, 68, 17, 73],\n",
       "  [86, 3, 32, 3, 62, 83, 68, 17, 73, 76],\n",
       "  [3, 32, 3, 62, 83, 68, 17, 73, 76, 72],\n",
       "  [32, 3, 62, 83, 68, 17, 73, 76, 72, 79],\n",
       "  [3, 62, 83, 68, 17, 73, 76, 72, 79, 71],\n",
       "  [62, 83, 68, 17, 73, 76, 72, 79, 71, 11],\n",
       "  [83, 68, 17, 73, 76, 72, 79, 71, 11, 73],\n",
       "  [68, 17, 73, 76, 72, 79, 71, 11, 73, 76],\n",
       "  [17, 73, 76, 72, 79, 71, 11, 73, 76, 72],\n",
       "  [73, 76, 72, 79, 71, 11, 73, 76, 72, 79],\n",
       "  [76, 72, 79, 71, 11, 73, 76, 72, 79, 71],\n",
       "  [72, 79, 71, 11, 73, 76, 72, 79, 71, 17],\n",
       "  [79, 71, 11, 73, 76, 72, 79, 71, 17, 81],\n",
       "  [71, 11, 73, 76, 72, 79, 71, 17, 81, 68],\n",
       "  [11, 73, 76, 72, 79, 71, 17, 81, 68, 80],\n",
       "  [73, 76, 72, 79, 71, 17, 81, 68, 80, 72],\n",
       "  [76, 72, 79, 71, 17, 81, 68, 80, 72, 15],\n",
       "  [72, 79, 71, 17, 81, 68, 80, 72, 15, 3],\n",
       "  [79, 71, 17, 81, 68, 80, 72, 15, 3, 87],\n",
       "  [71, 17, 81, 68, 80, 72, 15, 3, 87, 82],\n",
       "  [17, 81, 68, 80, 72, 15, 3, 87, 82, 66],\n",
       "  [81, 68, 80, 72, 15, 3, 87, 82, 66, 68],\n",
       "  [68, 80, 72, 15, 3, 87, 82, 66, 68, 85],\n",
       "  [80, 72, 15, 3, 87, 82, 66, 68, 85, 85],\n",
       "  [72, 15, 3, 87, 82, 66, 68, 85, 85, 82],\n",
       "  [15, 3, 87, 82, 66, 68, 85, 85, 82, 90],\n",
       "  [3, 87, 82, 66, 68, 85, 85, 82, 90, 66],\n",
       "  [87, 82, 66, 68, 85, 85, 82, 90, 66, 87],\n",
       "  [82, 66, 68, 85, 85, 82, 90, 66, 87, 92],\n",
       "  [66, 68, 85, 85, 82, 90, 66, 87, 92, 83],\n",
       "  [68, 85, 85, 82, 90, 66, 87, 92, 83, 72],\n",
       "  [85, 85, 82, 90, 66, 87, 92, 83, 72, 11],\n",
       "  [85, 82, 90, 66, 87, 92, 83, 72, 11, 73],\n",
       "  [82, 90, 66, 87, 92, 83, 72, 11, 73, 76],\n",
       "  [90, 66, 87, 92, 83, 72, 11, 73, 76, 72],\n",
       "  [66, 87, 92, 83, 72, 11, 73, 76, 72, 79],\n",
       "  [87, 92, 83, 72, 11, 73, 76, 72, 79, 71],\n",
       "  [92, 83, 72, 11, 73, 76, 72, 79, 71, 17],\n",
       "  [83, 72, 11, 73, 76, 72, 79, 71, 17, 71],\n",
       "  [72, 11, 73, 76, 72, 79, 71, 17, 71, 68],\n",
       "  [11, 73, 76, 72, 79, 71, 17, 71, 68, 87],\n",
       "  [73, 76, 72, 79, 71, 17, 71, 68, 87, 68],\n",
       "  [76, 72, 79, 71, 17, 71, 68, 87, 68, 55],\n",
       "  [72, 79, 71, 17, 71, 68, 87, 68, 55, 92],\n",
       "  [79, 71, 17, 71, 68, 87, 68, 55, 92, 83],\n",
       "  [71, 17, 71, 68, 87, 68, 55, 92, 83, 72],\n",
       "  [17, 71, 68, 87, 68, 55, 92, 83, 72, 12],\n",
       "  [71, 68, 87, 68, 55, 92, 83, 72, 12, 15],\n",
       "  [68, 87, 68, 55, 92, 83, 72, 12, 15, 3],\n",
       "  [87, 68, 55, 92, 83, 72, 12, 15, 3, 81],\n",
       "  [68, 55, 92, 83, 72, 12, 15, 3, 81, 88],\n",
       "  [55, 92, 83, 72, 12, 15, 3, 81, 88, 79],\n",
       "  [92, 83, 72, 12, 15, 3, 81, 88, 79, 79],\n",
       "  [83, 72, 12, 15, 3, 81, 88, 79, 79, 68],\n",
       "  [72, 12, 15, 3, 81, 88, 79, 79, 68, 69],\n",
       "  [12, 15, 3, 81, 88, 79, 79, 68, 69, 79],\n",
       "  [15, 3, 81, 88, 79, 79, 68, 69, 79, 72],\n",
       "  [3, 81, 88, 79, 79, 68, 69, 79, 72, 32],\n",
       "  [81, 88, 79, 79, 68, 69, 79, 72, 32, 73],\n",
       "  [88, 79, 79, 68, 69, 79, 72, 32, 73, 76],\n",
       "  [79, 79, 68, 69, 79, 72, 32, 73, 76, 72],\n",
       "  [79, 68, 69, 79, 72, 32, 73, 76, 72, 79],\n",
       "  [68, 69, 79, 72, 32, 73, 76, 72, 79, 71],\n",
       "  [69, 79, 72, 32, 73, 76, 72, 79, 71, 17],\n",
       "  [79, 72, 32, 73, 76, 72, 79, 71, 17, 81],\n",
       "  [72, 32, 73, 76, 72, 79, 71, 17, 81, 88],\n",
       "  [32, 73, 76, 72, 79, 71, 17, 81, 88, 79],\n",
       "  [73, 76, 72, 79, 71, 17, 81, 88, 79, 79],\n",
       "  [76, 72, 79, 71, 17, 81, 88, 79, 79, 68],\n",
       "  [72, 79, 71, 17, 81, 88, 79, 79, 68, 69],\n",
       "  [79, 71, 17, 81, 88, 79, 79, 68, 69, 79],\n",
       "  [71, 17, 81, 88, 79, 79, 68, 69, 79, 72],\n",
       "  [17, 81, 88, 79, 79, 68, 69, 79, 72, 12],\n",
       "  [81, 88, 79, 79, 68, 69, 79, 72, 12, 2],\n",
       "  [88, 79, 79, 68, 69, 79, 72, 12, 2, 3],\n",
       "  [79, 79, 68, 69, 79, 72, 12, 2, 3, 3],\n",
       "  [79, 68, 69, 79, 72, 12, 2, 3, 3, 3],\n",
       "  [68, 69, 79, 72, 12, 2, 3, 3, 3, 3],\n",
       "  [69, 79, 72, 12, 2, 3, 3, 3, 3, 3],\n",
       "  [79, 72, 12, 2, 3, 3, 3, 3, 3, 3],\n",
       "  [72, 12, 2, 3, 3, 3, 3, 3, 3, 3],\n",
       "  [12, 2, 3, 3, 3, 3, 3, 3, 3, 3],\n",
       "  [2, 3, 3, 3, 3, 3, 3, 3, 3, 3],\n",
       "  [3, 3, 3, 3, 3, 3, 3, 3, 3, 3],\n",
       "  [3, 3, 3, 3, 3, 3, 3, 3, 3, 3],\n",
       "  [3, 3, 3, 3, 3, 3, 3, 3, 3, 3],\n",
       "  [3, 3, 3, 3, 3, 3, 3, 3, 3, 3],\n",
       "  [3, 3, 3, 3, 3, 3, 3, 3, 3, 3],\n",
       "  [3, 3, 3, 3, 3, 3, 3, 3, 3, 73],\n",
       "  [3, 3, 3, 3, 3, 3, 3, 3, 73, 82],\n",
       "  [3, 3, 3, 3, 3, 3, 3, 73, 82, 85],\n",
       "  [3, 3, 3, 3, 3, 3, 73, 82, 85, 3],\n",
       "  [3, 3, 3, 3, 3, 73, 82, 85, 3, 73],\n",
       "  [3, 3, 3, 3, 73, 82, 85, 3, 73, 76],\n",
       "  [3, 3, 3, 73, 82, 85, 3, 73, 76, 72],\n",
       "  [3, 3, 73, 82, 85, 3, 73, 76, 72, 79],\n",
       "  [3, 73, 82, 85, 3, 73, 76, 72, 79, 71],\n",
       "  [73, 82, 85, 3, 73, 76, 72, 79, 71, 3],\n",
       "  [82, 85, 3, 73, 76, 72, 79, 71, 3, 76],\n",
       "  [85, 3, 73, 76, 72, 79, 71, 3, 76, 81],\n",
       "  [3, 73, 76, 72, 79, 71, 3, 76, 81, 3],\n",
       "  [73, 76, 72, 79, 71, 3, 76, 81, 3, 86],\n",
       "  [76, 72, 79, 71, 3, 76, 81, 3, 86, 70],\n",
       "  [72, 79, 71, 3, 76, 81, 3, 86, 70, 75],\n",
       "  [79, 71, 3, 76, 81, 3, 86, 70, 75, 72],\n",
       "  [71, 3, 76, 81, 3, 86, 70, 75, 72, 80],\n",
       "  [3, 76, 81, 3, 86, 70, 75, 72, 80, 68],\n",
       "  [76, 81, 3, 86, 70, 75, 72, 80, 68, 64],\n",
       "  [81, 3, 86, 70, 75, 72, 80, 68, 64, 2],\n",
       "  [3, 86, 70, 75, 72, 80, 68, 64, 2, 3],\n",
       "  [86, 70, 75, 72, 80, 68, 64, 2, 3, 3],\n",
       "  [70, 75, 72, 80, 68, 64, 2, 3, 3, 3],\n",
       "  [75, 72, 80, 68, 64, 2, 3, 3, 3, 3],\n",
       "  [72, 80, 68, 64, 2, 3, 3, 3, 3, 85],\n",
       "  [80, 68, 64, 2, 3, 3, 3, 3, 85, 72],\n",
       "  [68, 64, 2, 3, 3, 3, 3, 85, 72, 87],\n",
       "  [64, 2, 3, 3, 3, 3, 85, 72, 87, 88],\n",
       "  [2, 3, 3, 3, 3, 85, 72, 87, 88, 85],\n",
       "  [3, 3, 3, 3, 85, 72, 87, 88, 85, 81],\n",
       "  [3, 3, 3, 85, 72, 87, 88, 85, 81, 3],\n",
       "  [3, 3, 85, 72, 87, 88, 85, 81, 3, 83],\n",
       "  [3, 85, 72, 87, 88, 85, 81, 3, 83, 68],\n",
       "  [85, 72, 87, 88, 85, 81, 3, 83, 68, 17],\n",
       "  [72, 87, 88, 85, 81, 3, 83, 68, 17, 86],\n",
       "  [87, 88, 85, 81, 3, 83, 68, 17, 86, 70],\n",
       "  [88, 85, 81, 3, 83, 68, 17, 86, 70, 75],\n",
       "  [85, 81, 3, 83, 68, 17, 86, 70, 75, 72],\n",
       "  [81, 3, 83, 68, 17, 86, 70, 75, 72, 80],\n",
       "  [3, 83, 68, 17, 86, 70, 75, 72, 80, 68],\n",
       "  [83, 68, 17, 86, 70, 75, 72, 80, 68, 11],\n",
       "  [68, 17, 86, 70, 75, 72, 80, 68, 11, 73],\n",
       "  [17, 86, 70, 75, 72, 80, 68, 11, 73, 76],\n",
       "  [86, 70, 75, 72, 80, 68, 11, 73, 76, 72],\n",
       "  [70, 75, 72, 80, 68, 11, 73, 76, 72, 79],\n",
       "  [75, 72, 80, 68, 11, 73, 76, 72, 79, 71],\n",
       "  [72, 80, 68, 11, 73, 76, 72, 79, 71, 86],\n",
       "  [80, 68, 11, 73, 76, 72, 79, 71, 86, 12]],\n",
       " 'labels': [71,\n",
       "  72,\n",
       "  73,\n",
       "  3,\n",
       "  87,\n",
       "  82,\n",
       "  66,\n",
       "  68,\n",
       "  85,\n",
       "  85,\n",
       "  82,\n",
       "  90,\n",
       "  66,\n",
       "  86,\n",
       "  70,\n",
       "  75,\n",
       "  72,\n",
       "  80,\n",
       "  68,\n",
       "  11,\n",
       "  86,\n",
       "  70,\n",
       "  75,\n",
       "  72,\n",
       "  80,\n",
       "  68,\n",
       "  12,\n",
       "  29,\n",
       "  2,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  3,\n",
       "  38,\n",
       "  82,\n",
       "  81,\n",
       "  89,\n",
       "  72,\n",
       "  85,\n",
       "  87,\n",
       "  3,\n",
       "  68,\n",
       "  3,\n",
       "  86,\n",
       "  70,\n",
       "  75,\n",
       "  72,\n",
       "  80,\n",
       "  68,\n",
       "  3,\n",
       "  73,\n",
       "  85,\n",
       "  82,\n",
       "  80,\n",
       "  3,\n",
       "  54,\n",
       "  83,\n",
       "  68,\n",
       "  85,\n",
       "  78,\n",
       "  3,\n",
       "  87,\n",
       "  82,\n",
       "  3,\n",
       "  36,\n",
       "  85,\n",
       "  85,\n",
       "  82,\n",
       "  90,\n",
       "  2,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  2,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  76,\n",
       "  80,\n",
       "  83,\n",
       "  82,\n",
       "  85,\n",
       "  87,\n",
       "  3,\n",
       "  83,\n",
       "  92,\n",
       "  68,\n",
       "  85,\n",
       "  85,\n",
       "  82,\n",
       "  90,\n",
       "  3,\n",
       "  68,\n",
       "  86,\n",
       "  3,\n",
       "  83,\n",
       "  68,\n",
       "  2,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  73,\n",
       "  76,\n",
       "  72,\n",
       "  79,\n",
       "  71,\n",
       "  86,\n",
       "  3,\n",
       "  32,\n",
       "  3,\n",
       "  62,\n",
       "  83,\n",
       "  68,\n",
       "  17,\n",
       "  73,\n",
       "  76,\n",
       "  72,\n",
       "  79,\n",
       "  71,\n",
       "  11,\n",
       "  73,\n",
       "  76,\n",
       "  72,\n",
       "  79,\n",
       "  71,\n",
       "  17,\n",
       "  81,\n",
       "  68,\n",
       "  80,\n",
       "  72,\n",
       "  15,\n",
       "  3,\n",
       "  87,\n",
       "  82,\n",
       "  66,\n",
       "  68,\n",
       "  85,\n",
       "  85,\n",
       "  82,\n",
       "  90,\n",
       "  66,\n",
       "  87,\n",
       "  92,\n",
       "  83,\n",
       "  72,\n",
       "  11,\n",
       "  73,\n",
       "  76,\n",
       "  72,\n",
       "  79,\n",
       "  71,\n",
       "  17,\n",
       "  71,\n",
       "  68,\n",
       "  87,\n",
       "  68,\n",
       "  55,\n",
       "  92,\n",
       "  83,\n",
       "  72,\n",
       "  12,\n",
       "  15,\n",
       "  3,\n",
       "  81,\n",
       "  88,\n",
       "  79,\n",
       "  79,\n",
       "  68,\n",
       "  69,\n",
       "  79,\n",
       "  72,\n",
       "  32,\n",
       "  73,\n",
       "  76,\n",
       "  72,\n",
       "  79,\n",
       "  71,\n",
       "  17,\n",
       "  81,\n",
       "  88,\n",
       "  79,\n",
       "  79,\n",
       "  68,\n",
       "  69,\n",
       "  79,\n",
       "  72,\n",
       "  12,\n",
       "  2,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  73,\n",
       "  82,\n",
       "  85,\n",
       "  3,\n",
       "  73,\n",
       "  76,\n",
       "  72,\n",
       "  79,\n",
       "  71,\n",
       "  3,\n",
       "  76,\n",
       "  81,\n",
       "  3,\n",
       "  86,\n",
       "  70,\n",
       "  75,\n",
       "  72,\n",
       "  80,\n",
       "  68,\n",
       "  64,\n",
       "  2,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  85,\n",
       "  72,\n",
       "  87,\n",
       "  88,\n",
       "  85,\n",
       "  81,\n",
       "  3,\n",
       "  83,\n",
       "  68,\n",
       "  17,\n",
       "  86,\n",
       "  70,\n",
       "  75,\n",
       "  72,\n",
       "  80,\n",
       "  68,\n",
       "  11,\n",
       "  73,\n",
       "  76,\n",
       "  72,\n",
       "  79,\n",
       "  71,\n",
       "  86,\n",
       "  12,\n",
       "  1]}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "process(datasets[8], tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "96f119662322ffb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.425382Z",
     "start_time": "2025-08-23T14:41:15.415406Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['repo', 'path', 'func_name', 'original_string', 'language', 'code', 'code_tokens', 'docstring', 'docstring_tokens', 'sha', 'url', 'partition']\n"
     ]
    }
   ],
   "source": [
    "tokenized = datasets.train_test_split(test_size=0.1, seed=1024, shuffle=True)\n",
    "print(datasets.column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ace9ce4bb6729477",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.473066Z",
     "start_time": "2025-08-23T14:41:15.446771Z"
    }
   },
   "outputs": [],
   "source": [
    "f = lambda x: process(x, tokenizer)\n",
    "# 当 batch=True 会传入一个列表\n",
    "tokenized = tokenized.map(f, batched=True, remove_columns=datasets.column_names)\n",
    "tokenized.set_format(type='torch', device='cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b730319c2d3d6db3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.484082Z",
     "start_time": "2025-08-23T14:41:15.481573Z"
    }
   },
   "outputs": [],
   "source": [
    "batch_size = 1000\n",
    "learning_rate = 0.01\n",
    "eval_iters = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8c2db4b68fbc23c3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.495480Z",
     "start_time": "2025-08-23T14:41:15.492631Z"
    }
   },
   "outputs": [],
   "source": [
    "train_loader = DataLoader(tokenized['train'], batch_size=batch_size, shuffle=True)\n",
    "test_loader = DataLoader(tokenized['test'], batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "e1e1142ac30b1ed9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:15.732369Z",
     "start_time": "2025-08-23T14:41:15.510082Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'inputs': tensor([[ 2,  3,  3,  ...,  3,  3, 85],\n",
       "         [72, 87, 36,  ..., 72, 81, 87],\n",
       "         [68, 81,  3,  ...,  3, 93, 76],\n",
       "         ...,\n",
       "         [ 3,  3,  3,  ..., 33, 33,  3],\n",
       "         [80, 51, 68,  ..., 76, 82, 81],\n",
       "         [ 5,  2,  3,  ..., 82, 88, 81]], device='cuda:0'),\n",
       " 'labels': tensor([72, 86, 83, 88, 72,  3,  3,  3, 38, 82, 82,  3, 87, 73,  3, 81, 54,  3,\n",
       "         12, 92, 39, 72, 87, 79, 83, 16,  3,  3, 29, 76,  3, 70, 88, 80, 81, 12,\n",
       "          5, 72, 26, 86, 17, 85,  3,  2,  3, 70,  3, 85,  2, 72, 69, 17, 80, 68,\n",
       "         76, 11, 15,  3, 76, 87,  3, 68,  2,  3, 38, 70, 81, 51, 87, 16,  3, 71,\n",
       "          5, 87,  3,  3, 85, 78,  3, 72,  3, 72, 72, 87, 81, 66, 15,  3, 87, 76,\n",
       "          2,  3,  2, 80, 72, 89, 75, 87, 85, 17, 72, 49, 72, 85, 87,  3, 86, 76,\n",
       "         72,  3,  2, 85,  1,  3,  3,  3,  3,  3, 82,  3,  3, 72,  3, 68,  3,  3,\n",
       "          3, 11, 33,  3, 81, 38, 70,  3, 72, 66, 81, 67, 92,  3, 79, 86,  3, 64,\n",
       "         85,  3, 82, 16, 89, 73, 73, 82, 71, 29, 72, 87, 75, 68, 14, 71, 75, 72,\n",
       "         85, 72, 72, 72, 72, 72,  3, 86,  3, 85, 71,  3,  3,  3, 82, 15, 74, 79,\n",
       "         82, 17, 89, 62, 11, 76,  3,  3, 79, 66, 80,  2,  3, 32, 80,  3, 12, 11,\n",
       "          3,  3, 85, 73,  3, 76,  3, 38,  3,  3, 15, 91,  3, 72, 68,  3, 81, 11,\n",
       "         10, 32, 86,  3, 72, 87, 76,  3, 17,  3,  2,  3,  2, 72,  3, 87, 20, 84,\n",
       "         68, 39, 87,  2, 86, 72, 44, 12, 87, 85,  3, 17, 58,  3, 16, 92, 85,  3,\n",
       "         72, 88,  3,  3,  3, 36, 72, 72, 72,  3, 85,  3,  3, 86, 86,  3, 87,  3,\n",
       "         72, 83,  5, 68, 79, 82, 81,  3, 76, 82, 29,  3, 15, 12, 85, 20, 87, 87,\n",
       "         68, 72, 25, 80, 86, 73, 76, 70, 40, 85, 10,  3,  3, 73, 72,  2,  3, 12,\n",
       "         73,  3,  3,  2, 79, 83, 86, 94, 81, 72,  3, 86,  5,  3, 73,  3, 15, 59,\n",
       "         82, 12, 75, 73, 71,  3, 81,  3, 85,  1, 73,  3,  3, 71, 70,  3, 66, 76,\n",
       "          3, 68, 29, 86,  3,  3, 64, 74, 87, 76, 87, 72, 85, 76,  3,  3,  3, 68,\n",
       "         80, 19, 82,  3, 39, 72, 76, 19,  3, 33, 12,  3, 33, 73, 89,  3, 68,  3,\n",
       "          5, 11, 44, 83, 86, 87, 85, 17, 79, 71,  3,  3,  3, 11,  3,  3,  3, 76,\n",
       "         85, 80,  3,  3, 72,  3, 67,  3, 16,  3, 17,  2,  2, 87,  3,  3,  3, 87,\n",
       "         82, 87, 82,  3,  3,  3, 20,  2, 86, 88, 68, 86, 73, 82, 12, 85, 76,  3,\n",
       "         11, 76,  5, 72, 72,  3, 90, 66,  3,  3, 76, 83,  3,  2, 81, 75, 71, 83,\n",
       "         68,  6, 68,  3, 86, 53,  3, 72, 66, 33, 88, 79, 83, 72, 85, 75, 87, 55,\n",
       "         76,  3, 92,  3,  3, 29, 80,  3, 87, 72, 79,  3, 10,  5, 71, 85,  3, 85,\n",
       "         68,  3, 82,  3, 81, 76, 66,  3, 33, 16, 76,  3,  3, 64, 75,  2, 72, 71,\n",
       "         76,  3,  3,  3, 72, 71,  3, 86, 82,  3, 69, 72, 44,  3, 79, 82, 83,  3,\n",
       "          2, 71, 76,  3,  3, 69, 81,  3,  3, 76, 15,  3, 74,  2, 17, 87, 71, 67,\n",
       "         85, 68, 72,  3,  3, 93, 79, 79, 57, 56, 72, 82, 87, 83, 29,  5,  3,  3,\n",
       "         68,  3, 83, 51, 71, 72, 95, 82, 68,  3,  3, 17, 87,  3, 85, 68, 32, 82,\n",
       "          3, 88,  2, 82,  3, 87, 81, 68,  3, 38, 80, 76,  3, 33, 86,  3,  3, 92,\n",
       "         39,  3, 76, 76, 87, 88, 32, 88,  3, 72,  3,  3, 72,  3,  3,  3, 72, 82,\n",
       "         82,  3,  3, 83,  2, 62, 82, 82, 17, 74,  3, 87, 15,  3, 80,  2, 72, 87,\n",
       "          3, 72, 86,  3, 72, 17,  3, 92, 82,  3, 80, 91, 87, 88, 85,  3, 82, 33,\n",
       "          3,  2, 87, 76, 72,  2,  3, 85, 12, 76, 68, 81,  3,  3, 76,  3, 73, 82,\n",
       "          3,  3, 86,  3,  3, 85, 68, 73, 86,  3,  3,  3, 82,  3, 68, 87, 86, 72,\n",
       "         72, 53,  3, 72, 81, 23, 68, 86, 87, 86, 71, 20,  3,  3, 76,  3,  3, 39,\n",
       "         69, 72, 68,  3, 68, 68, 41, 85, 82, 85, 23, 23,  3, 29,  3, 51, 78,  3,\n",
       "          3, 29,  3, 70, 80,  3, 58,  3,  3, 87, 82, 88, 51, 72, 76,  3, 87, 86,\n",
       "         66,  3, 85, 32, 72, 72,  3,  3,  3, 82, 33, 49,  2, 82, 79, 53, 71, 80,\n",
       "         80, 86, 66, 72, 89, 10,  3,  3, 33, 70,  3,  3, 68, 85, 72, 29,  3, 68,\n",
       "         66,  5,  3,  3,  3, 72, 88, 75, 70, 82,  5, 15, 82, 16, 76,  3, 14, 86,\n",
       "         72, 87,  3,  3,  3, 17,  3, 56,  3,  3, 68,  3,  3, 12, 79, 81,  3, 73,\n",
       "         33,  3, 66, 76, 29,  5, 71,  3, 89,  2, 74,  3, 81,  3, 66, 68,  3, 11,\n",
       "         82,  3, 80, 82, 81,  3, 87, 83,  3, 92, 70,  3, 86, 87,  3, 92,  3, 12,\n",
       "          3,  3, 72,  3,  3,  3, 71, 17, 95, 85, 87,  3, 20, 69, 68, 82, 12, 83,\n",
       "         86, 72, 68, 41, 81, 79, 75,  3,  3, 24,  3, 72,  3, 80, 86, 85, 82, 70,\n",
       "          3, 85, 81, 87, 17,  3, 14, 87, 15,  3, 72, 20, 47, 86, 68,  3,  3,  3,\n",
       "         85, 41, 16, 87, 73, 20,  3, 68, 74,  3, 72, 83, 56, 11, 20, 80, 76, 64,\n",
       "         64,  3, 80, 77, 86,  3, 40,  3, 73,  3, 55, 10, 71, 11, 12, 72, 83, 85,\n",
       "          3, 81, 82,  3, 86, 66, 87,  3, 92,  3, 72, 79,  3, 72, 82, 38, 80,  3,\n",
       "         71, 72, 83,  3, 66, 87, 85, 79, 71,  3, 87, 87, 90, 75, 53,  5,  3, 76,\n",
       "         76, 29, 17, 75,  3, 33,  3, 75, 72, 80,  3,  3, 72, 72,  5, 48,  3,  3,\n",
       "         85,  3, 68, 89,  3,  8, 10, 85, 86, 71], device='cuda:0')}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(iter(train_loader))"
   ]
  },
  {
   "cell_type": "code",
   "id": "98c8b9a471531b3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T15:18:52.329949Z",
     "start_time": "2025-08-23T15:18:52.324950Z"
    }
   },
   "source": [
    "class CharMLP(nn.Module):\n",
    "\n",
    "    def __init__(self, vs):\n",
    "        # vs 字典大小\n",
    "        super().__init__()\n",
    "        self.emb = nn.Embedding(vs, 30)\n",
    "        self.hidden1 = nn.Linear(10 * 30, 200)\n",
    "        self.hidden2 = nn.Linear(200, 100)\n",
    "        self.lm = nn.Linear(100, vs)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        B = x.shape[0]\n",
    "        emb = self.emb(x)  #  (B, 10, 30)\n",
    "        h = emb.view(B, -1)  # (B, 300)\n",
    "        h = self.hidden1(h)\n",
    "        h = F.relu(h)  # (B, 200)\n",
    "        h = self.hidden2(h)\n",
    "        h = F.relu(h)  # (B, 100)\n",
    "        out = self.lm(h)  # (B, vs)\n",
    "        return out\n",
    "\n",
    "\"\"\"\n",
    "或者一开始送入向量也可以\n",
    "class CharMLP(nn.Module):\n",
    "    def __init__(self, vs):\n",
    "        super().__init__()\n",
    "        self.emb = nn.Embedding(vs, 30)\n",
    "        # 对每个位置单独处理\n",
    "        self.hidden1 = nn.Linear(30, 200)\n",
    "        self.hidden2 = nn.Linear(200, 100)\n",
    "        # 最后用一个线性层聚合所有位置信息\n",
    "        self.lm = nn.Linear(10 * 100, vs)  # 注意这里还是要展平\n",
    "\n",
    "    def forward(self, x):\n",
    "        B = x.shape[0]\n",
    "        emb = self.emb(x)  # (B, 10, 30)\n",
    "        h = F.relu(self.hidden1(emb))  # (B, 10, 200)\n",
    "        h = F.relu(self.hidden2(h))  # (B, 10, 100)\n",
    "        h = h.view(B, -1)  # (B, 1000) ← 最终还是需要展平\n",
    "        logits = self.lm(h)  # (B, vs)\n",
    "        return logits\n",
    "\"\"\""
   ],
   "outputs": [],
   "execution_count": 44
  },
  {
   "cell_type": "code",
   "id": "24334845c37be187",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T15:18:53.360522Z",
     "start_time": "2025-08-23T15:18:53.352971Z"
    }
   },
   "source": [
    "model = CharMLP(len(tokenizer.char2ind)).to('cuda')\n",
    "model"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CharMLP(\n",
       "  (emb): Embedding(99, 30)\n",
       "  (hidden1): Linear(in_features=30, out_features=200, bias=True)\n",
       "  (hidden2): Linear(in_features=200, out_features=100, bias=True)\n",
       "  (lm): Linear(in_features=1000, out_features=99, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 45
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "dfa95db21c9bff24",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:16.349893Z",
     "start_time": "2025-08-23T14:41:15.806587Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': 4.5956830978393555, 'test': 4.594418525695801}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def estimate_loss(model):\n",
    "    re = {}\n",
    "    # 将模型切换至评估模式\n",
    "    model.eval()\n",
    "    re['train'] = _loss(model, train_loader)\n",
    "    re['test'] = _loss(model, test_loader)\n",
    "    # 将模型切换至训练模式\n",
    "    model.train()\n",
    "    return re\n",
    "\n",
    "@torch.no_grad()\n",
    "def _loss(model, data_loader):\n",
    "    \"\"\"\n",
    "    计算模型在不同数据集下面的评估指标\n",
    "    \"\"\"\n",
    "    loss = []\n",
    "    data_iter= iter(data_loader)\n",
    "    # 随机使用多个批量数据来预估模型效果\n",
    "    for k in range(eval_iters):\n",
    "        data = next(data_iter, None)\n",
    "        if data is None:\n",
    "            data_iter = iter(data_loader)\n",
    "            data = next(data_iter, None)\n",
    "        inputs, labels = data['inputs'], data['labels']\n",
    "        logits = model(inputs)\n",
    "        loss.append(F.cross_entropy(logits, labels).item())\n",
    "    return torch.tensor(loss).mean().item()\n",
    "\n",
    "estimate_loss(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d978f68681cb85c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:16.364145Z",
     "start_time": "2025-08-23T14:41:16.360438Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_model(model, optimizer, epochs=10):\n",
    "    # 记录模型在训练集上的模型损失\n",
    "    lossi = []\n",
    "    for epoch in range(epochs):\n",
    "        for i, data in enumerate(train_loader, 0):\n",
    "            inputs, labels = data['inputs'], data['labels']\n",
    "            optimizer.zero_grad()\n",
    "            logits = model(inputs)\n",
    "            loss = F.cross_entropy(logits, labels)\n",
    "            lossi.append(loss.item())\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "        # 评估模型，并输出结果\n",
    "        stats = estimate_loss(model)\n",
    "        train_loss = f'train loss {stats[\"train\"]:.4f}'\n",
    "        test_loss = f'test loss {stats[\"test\"]:.4f}'\n",
    "        print(f'epoch {epoch:>2}: {train_loss}, {test_loss}')\n",
    "    return lossi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "d6844f03d83fe81",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:16.397311Z",
     "start_time": "2025-08-23T14:41:16.393265Z"
    }
   },
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def generate(model, context, tokenizer, max_new_tokens=300):\n",
    "    out = []\n",
    "    model.eval()\n",
    "    for _ in range(max_new_tokens):\n",
    "        logits = model(context)   # (1, 99)\n",
    "        probs = F.softmax(logits, dim=-1)   # (1, 99)\n",
    "        # 随机生成文本\n",
    "        ix = torch.multinomial(probs, num_samples=1)  # (1, 1)\n",
    "        # 更新\n",
    "        context = torch.concat((context[:, 1:], ix), dim=-1)\n",
    "        out.append(ix.item())\n",
    "        if out[-1] == tokenizer.end_ind:\n",
    "            break\n",
    "    model.train()\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "id": "2f839f2940c939e9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T15:18:59.583241Z",
     "start_time": "2025-08-23T15:18:59.538194Z"
    }
   },
   "source": [
    "context = torch.zeros((1, 10), dtype=torch.long, device='cuda')\n",
    "print(''.join(tokenizer.decode(generate(model, context, tokenizer))))"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oY[ZCJVEod<|e|>\n"
     ]
    }
   ],
   "execution_count": 47
  },
  {
   "cell_type": "code",
   "id": "840ac850bba84223",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T15:21:21.475843Z",
     "start_time": "2025-08-23T15:19:04.497419Z"
    }
   },
   "source": [
    "l = train_model(model, optim.Adam(model.parameters(), lr=learning_rate))"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch  0: train loss 1.9259, test loss 2.0615\n",
      "epoch  1: train loss 1.9106, test loss 2.0043\n",
      "epoch  2: train loss 1.8638, test loss 2.0178\n",
      "epoch  3: train loss 1.8432, test loss 2.0386\n",
      "epoch  4: train loss 1.8413, test loss 1.9916\n",
      "epoch  5: train loss 1.8230, test loss 1.9883\n",
      "epoch  6: train loss 1.8371, test loss 1.9948\n",
      "epoch  7: train loss 1.7987, test loss 2.0343\n",
      "epoch  8: train loss 1.8180, test loss 1.9629\n",
      "epoch  9: train loss 1.8358, test loss 1.9717\n"
     ]
    }
   ],
   "execution_count": 48
  },
  {
   "cell_type": "code",
   "id": "8c32acc776dab5fa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T15:21:31.993845Z",
     "start_time": "2025-08-23T15:21:31.755403Z"
    }
   },
   "source": [
    "context = torch.zeros((1, 10), dtype=torch.long, device='cuda')\n",
    "print(''.join(tokenizer.decode(generate(model, context, tokenizer))))"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def _sct3, 2.1\n",
      "        O s  = SParkConlactumpas, pardifiD']\n",
      "    >>> rTepot  = sclitson= or.dCllet: ac.Sor b Vactoron S{ric(morndef.Caluntsine m  m aur :: ly ve  nos -= g\n",
      "\n",
      "        uer colunclinumll ne, ce_forite\n",
      "        hist ty naldeldetite tessoumperoummoyFeame\n",
      "    Recurnso n re.dug aveluns if None:\n"
     ]
    }
   ],
   "execution_count": 55
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6485db39f02a3698",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:41:29.661527500Z",
     "start_time": "2025-08-23T14:07:23.902006Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x2165e5420d0>]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAATBVJREFUeJzt3XlYU1f+BvD3JkAAIWHfZBEFRcR9xd2KWx2rXazj2Gr32tFW2047tftOf9M6baeL1jqt3dTaxWXcqWtdcUMBFUVRUBZBhACyJvf3R8glIWFT4Ip5P8+TR3Jzk5zcWvJ6zvecI4iiKIKIiIhIJgq5G0BERES2jWGEiIiIZMUwQkRERLJiGCEiIiJZMYwQERGRrBhGiIiISFYMI0RERCQrhhEiIiKSlZ3cDWgMvV6PzMxMuLq6QhAEuZtDREREjSCKIoqKihAQEACFou7+jzYRRjIzMxEUFCR3M4iIiOgGZGRkIDAwsM7H20QYcXV1BWD4MGq1WubWEBERUWNotVoEBQVJ3+N1aRNhxDg0o1arGUaIiIjamIZKLFjASkRERLJiGCEiIiJZMYwQERGRrBhGiIiISFYMI0RERCQrhhEiIiKSFcMIERERyYphhIiIiGTFMEJERESyYhghIiIiWTGMEBERkawYRoiIiEhWbWKjvJby3z1pyMi/jr8OCEKEHzfgIyIikoNN94xsOJGJZfsuIP3qdbmbQkREZLNsOowoqrc01osyN4SIiMiGNSmMLFq0CD169IBarYZarUZ0dDQ2bdpU5/nLli2DIAhmN0dHx5tudHMxhhFRZBohIiKSS5NqRgIDA/HBBx8gPDwcoijiu+++w+TJk3Hs2DF069bN6nPUajVSUlKk+0J1ALgVGJvCnhEiIiL5NCmMTJo0yez+e++9h0WLFuHAgQN1hhFBEODn53fjLWxBxp4RHXtGiIiIZHPDNSM6nQ4rV65ESUkJoqOj6zyvuLgYISEhCAoKwuTJk5GcnNzga5eXl0Or1ZrdWoJSwWEaIiIiuTU5jCQmJsLFxQUqlQqzZ8/G6tWrERkZafXcLl264JtvvsHatWvx448/Qq/XY/Dgwbh06VK97xEbGwuNRiPdgoKCmtrMRqkZpmEYISIikosgNrFboKKiAunp6SgsLMSvv/6KpUuXYteuXXUGElOVlZXo2rUrpk+fjnfeeafO88rLy1FeXi7d12q1CAoKQmFhIdTq5lsPZNY38dh1JhcLp/bEvX0Dm+11iYiIyPD9rdFoGvz+bvKiZw4ODggLCwMA9O3bF4cOHcKnn36Kr776qsHn2tvbo3fv3khNTa33PJVKBZVK1dSmNZmCPSNERESyu+l1RvR6vVkvRn10Oh0SExPh7+9/s2/bLGqm9srcECIiIhvWpJ6RBQsWYMKECQgODkZRURGWL1+OnTt3YsuWLQCAmTNnon379oiNjQUAvP322xg0aBDCwsJQUFCADz/8EBcvXsRjjz3W/J/kBgicTUNERCS7JoWRK1euYObMmcjKyoJGo0GPHj2wZcsWjBkzBgCQnp4OhaKms+XatWt4/PHHkZ2dDXd3d/Tt2xf79u1rVH1Ja1BWN5XDNERERPJpcgGrHBpbANNUT/14BJuSsvHOlCg8OCik2V6XiIiIGv/9zb1pwHVGiIiI5GTTYURaZ4TrwRMREcnGpsNIzXLwMjeEiIjIhtl4GDH8yWEaIiIi+dh2GKlOI5xNQ0REJB/bDiOCMYzI3BAiIiIbZuNhxPAne0aIiIjkY+NhhMvBExERyc2mw4i0HDzHaYiIiGRj02GEy8ETERHJz6bDCAtYiYiI5McwAq4zQkREJCebDiMCZ9MQERHJzqbDCIdpiIiI5GfjYcTwJzfKIyIiko9thxEuB09ERCQ72w4jHKYhIiKSnY2HEcOf7BkhIiKSj42HES4HT0REJDebDiOCwJoRIiIiudl0GDEO03BvGiIiIvnYdBhRsoCViIhIdjYdRoxTe7kcPBERkXxsOoxwOXgiIiL52XQY4TojRERE8rPxMGL4k8vBExERycfGwwin9hIREcmNYQQcpiEiIpKTjYcRw5/sGSEiIpKPbYcRBZeDJyIikptNhxEuB09ERCQ/mw4jXA6eiIhIfjYdRrgcPBERkfxsOowYZ9NwOXgiIiL52HQY4XLwRERE8rPpMMJ1RoiIiORn22Gk+tOzZ4SIiEg+th1GOLWXiIhIdgwjAPR6mRtCRERkwxhGwJ4RIiIiOdl4GDH8ySxCREQkH5sOI1wOnoiISH42HUak5eAZRoiIiGRj42GE64wQERHJzabDiFLB5eCJiIjkZtNhhMvBExERyc+mwwjXGSEiIpIfwwjYM0JERCQnGw8jhj8ZRoiIiORj02FE4GwaIiIi2dl0GDHOpmHPCBERkXxsOoxwOXgiIiL52XQY4XLwRERE8rPpMMICViIiIvnZeBjhOiNERERys+kwwgJWIiIi+dl0GOFy8ERERPJrUhhZtGgRevToAbVaDbVajejoaGzatKne5/zyyy+IiIiAo6Mjunfvjo0bN95Ug5sTd+0lIiKSX5PCSGBgID744AMcOXIEhw8fxh133IHJkycjOTnZ6vn79u3D9OnT8eijj+LYsWOYMmUKpkyZgqSkpGZp/M0yhhHu2ktERCQfQbzJb2IPDw98+OGHePTRRy0emzZtGkpKSrB+/Xrp2KBBg9CrVy8sXry40e+h1Wqh0WhQWFgItVp9M801czanCGM+3g2Pdg44+tqYZntdIiIiavz39w3XjOh0OqxcuRIlJSWIjo62es7+/fsRExNjdmzcuHHYv39/va9dXl4OrVZrdmsJxnVGdBynISIikk2Tw0hiYiJcXFygUqkwe/ZsrF69GpGRkVbPzc7Ohq+vr9kxX19fZGdn1/sesbGx0Gg00i0oKKipzWwUzqYhIiKSX5PDSJcuXZCQkICDBw/iqaeewqxZs3Dy5MlmbdSCBQtQWFgo3TIyMpr19Y24HDwREZH87Jr6BAcHB4SFhQEA+vbti0OHDuHTTz/FV199ZXGun58fcnJyzI7l5OTAz8+v3vdQqVRQqVRNbVqTKbgcPBERkexuep0RvV6P8vJyq49FR0dj27ZtZsfi4uLqrDFpbVxnhIiISH5N6hlZsGABJkyYgODgYBQVFWH58uXYuXMntmzZAgCYOXMm2rdvj9jYWADAvHnzMGLECCxcuBATJ07EypUrcfjwYSxZsqT5P8kN4HLwRERE8mtSGLly5QpmzpyJrKwsaDQa9OjRA1u2bMGYMYZpsenp6VAoajpbBg8ejOXLl+PVV1/Fyy+/jPDwcKxZswZRUVHN+yluEIdpiIiI5HfT64y0hpZaZ+RKURkGvLcNCgE4Hzux2V6XiIiIWmGdkdsBl4MnIiKSH8NINT0TCRERkSxsOowYFz0DAN2tP1pFRER0W7LpMGJnGkbYM0JERCQLmw4jpj0jlTrO7yUiIpKDTYcRe2XNx2fPCBERkTxsOoyYdIygimGEiIhIFjYdRgRBkOpG2DNCREQkD5sOI0BN3QhrRoiIiORh82HEWDfCnhEiIiJ52HwYMfaMsGaEiIhIHjYfRlgzQkREJC+bDyOsGSEiIpKXzYcR1owQERHJy+bDCGtGiIiI5GXzYYQ1I0RERPKy+TDCmhEiIiJ52XwYsWPNCBERkawYRlgzQkREJCubDyPGYRqdjmGEiIhIDjYfRmp6RlgzQkREJAeGESWHaYiIiOTEMKJgASsREZGcbD6MSIuesWaEiIhIFjYfRlgzQkREJC+GEdaMEBERyYphhDUjREREsrL5MFKzHDzDCBERkRxsPozUbJTHmhEiIiI5MIywZoSIiEhWNh9GlMaaEQ7TEBERycLmw4hxmKaSPSNERESysPkwomTNCBERkaxsPozYs2aEiIhIVjYfRlgzQkREJC+bDyM1y8EzjBAREcnB5sOIknvTEBERycrmw4ixZoTLwRMREcnD5sOIsWakijUjREREsrD5MMKaESIiInnZfBhRMowQERHJyubDSE3NCAtYiYiI5GDzYYQ1I0RERPKy+TDCmhEiIiJ52XwYcbAzXIKKKg7TEBERycHmw4ijveESlFfpZG4JERGRbbL5MKKyUwIAyirZM0JERCQHhhE79owQERHJiWHEnj0jREREcmIYYc8IERGRrGw+jNQUsLJnhIiISA42H0ZqCljZM0JERCQHhhGTnhFR5MJnRERErc3mw4hjdQGrKAKVXBKeiIio1dl8GDEWsAJAGYtYiYiIWp3NhxEHpQKCYXsalHN6LxERUauz+TAiCILUO8IiViIiotbXpDASGxuL/v37w9XVFT4+PpgyZQpSUlLqfc6yZcsgCILZzdHR8aYa3dyMM2o4vZeIiKj1NSmM7Nq1C3PmzMGBAwcQFxeHyspKjB07FiUlJfU+T61WIysrS7pdvHjxphrd3LjwGRERkXzsmnLy5s2bze4vW7YMPj4+OHLkCIYPH17n8wRBgJ+f3421sBU4ckl4IiIi2dxUzUhhYSEAwMPDo97ziouLERISgqCgIEyePBnJycn1nl9eXg6tVmt2a0nsGSEiIpLPDYcRvV6P+fPnY8iQIYiKiqrzvC5duuCbb77B2rVr8eOPP0Kv12Pw4MG4dOlSnc+JjY2FRqORbkFBQTfazEYx9oywZoSIiKj13XAYmTNnDpKSkrBy5cp6z4uOjsbMmTPRq1cvjBgxAr///ju8vb3x1Vdf1fmcBQsWoLCwULplZGTcaDMbReoZ4WwaIiKiVtekmhGjuXPnYv369di9ezcCAwOb9Fx7e3v07t0bqampdZ6jUqmgUqlupGk3RMXN8oiIiGTTpJ4RURQxd+5crF69Gtu3b0doaGiT31Cn0yExMRH+/v5Nfm5LceRmeURERLJpUs/InDlzsHz5cqxduxaurq7Izs4GAGg0Gjg5OQEAZs6cifbt2yM2NhYA8Pbbb2PQoEEICwtDQUEBPvzwQ1y8eBGPPfZYM3+UG8eeESIiIvk0KYwsWrQIADBy5Eiz499++y0eeughAEB6ejoUipoOl2vXruHxxx9HdnY23N3d0bdvX+zbtw+RkZE31/JmJC16xqm9REREra5JYUQUG97VdufOnWb3P/74Y3z88cdNalRrc7TncvBERERysfm9aQAuB09ERCQnhhHU1IywZ4SIiKj1MYyAPSNERERyYhgBl4MnIiKSE8MIuFEeERGRnBhGwJ4RIiIiOTGMwDSMsGeEiIiotTGMwHSYhj0jRERErY1hBOwZISIikhPDCFjASkREJCeGEbCAlYiISE4MIwBU9twoj4iISC4MI6jZKI89I0RERK2PYQQmy8GzZ4SIiKjVMYyAs2mIiIjkxDCCmtk0FTo9dHpR5tYQERHZFoYR1PSMAEAFe0eIiIhaFcMIzMMIV2ElIiJqXQwjAOyUCtgpBABAGWfUEBERtSqGkWpO1XUjpRUMI0RERK2JYaSak0N1GOEwDRERUatiGKlmDCOsGSEiImpdDCPVjMM01zlMQ0RE1KoYRqpJwzQMI0RERK2KYaSaVMDKYRoiIqJWxTBSzZk9I0RERLJgGKnmyJ4RIiIiWTCMVGMBKxERkTwYRqo5c2ovERGRLBhGqjk6sGeEiIhIDgwj1TibhoiISB4MI9WkYRr2jBAREbUqhpFqLGAlIiKSB8NINScHOwAcpiEiImptDCPVpJoR9owQERG1KoaRak4OhktxvbJK5pYQERHZFoaRahonewBAYWmlzC0hIiKyLQwj1dycHQAABSUMI0RERK2JYaSae3UYKSqvQqVOL3NriIiIbAfDSDWNkz0EwfBzwXX2jhAREbUWhpFqSoUAtaOhbqTgeoXMrSEiIrIdDCMm3J0NYeQae0aIiIhaDcOICWMR6zX2jBAREbUahhETxp4RDtMQERG1HoYRE+5SzwiHaYiIiFoLw4gJDtMQERG1PoYRE9IwDRc+IyIiajUMIybc2lWvwlrKnhEiIqLWwjBiglN7iYiIWh/DiAljAStn0xAREbUehhETbuwZISIianUMIyZMe0ZEUZS5NURERLaBYcSEMYxU6kSUVOhkbg0REZFtYBgx4eSghMrOcEmulbBuhIiIqDUwjNTiUT299yrDCBERUatgGKklyMMZAHAhr0TmlhAREdkGhpFaOnm3AwCcZxghIiJqFU0KI7Gxsejfvz9cXV3h4+ODKVOmICUlpcHn/fLLL4iIiICjoyO6d++OjRs33nCDW1qoV3UYyS2WuSVERES2oUlhZNeuXZgzZw4OHDiAuLg4VFZWYuzYsSgpqbsXYd++fZg+fToeffRRHDt2DFOmTMGUKVOQlJR0041vCR29XAAAaewZISIiahWCeBMLauTm5sLHxwe7du3C8OHDrZ4zbdo0lJSUYP369dKxQYMGoVevXli8eHGj3ker1UKj0aCwsBBqtfpGm9so53KLMXrhLjg7KJH81jgIgtCi70dERHS7auz3903VjBQWFgIAPDw86jxn//79iImJMTs2btw47N+/v87nlJeXQ6vVmt1aS7CHM5QKAdcrdMjRlrfa+xIREdmqGw4jer0e8+fPx5AhQxAVFVXnednZ2fD19TU75uvri+zs7DqfExsbC41GI92CgoJutJlNZq9UILh6Rg3rRoiIiFreDYeROXPmICkpCStXrmzO9gAAFixYgMLCQumWkZHR7O9Rn45enFFDRETUWuxu5Elz587F+vXrsXv3bgQGBtZ7rp+fH3JycsyO5eTkwM/Pr87nqFQqqFSqG2laszDOqGERKxERUctrUs+IKIqYO3cuVq9eje3btyM0NLTB50RHR2Pbtm1mx+Li4hAdHd20lrYiP40jACCvmDUjRERELa1JPSNz5szB8uXLsXbtWri6ukp1HxqNBk5OTgCAmTNnon379oiNjQUAzJs3DyNGjMDChQsxceJErFy5EocPH8aSJUua+aM0H7WjPQBAW1opc0uIiIhuf03qGVm0aBEKCwsxcuRI+Pv7S7eff/5ZOic9PR1ZWVnS/cGDB2P58uVYsmQJevbsiV9//RVr1qypt+hVbmonQxgpZBghIiJqcU3qGWnMkiQ7d+60ODZ16lRMnTq1KW8lK7WT4bJoy6pkbgkREdHtj3vTWMFhGiIiotbDMGKFhsM0RERErYZhxApjzUh5lR5llTqZW0NERHR7YxixwlVlB+OWNEWsGyEiImpRDCNWKBQCXFWGIlYO1RAREbUshpE6GIdqvtt3ATnaMplbQ0REdPtiGKmDm7MhjPxw4CLe33hK5tYQERHdvhhG6vDgoBDp5/O53KOGiIiopTCM1GFa/2AsfqAvAKBK3/Bib0RERHRjGEbq4aM27BxcVMYiViIiopbCMFIPtaNhRg2n9xIREbUchpF6uFYvC19cXtWofXmIiIio6RhG6uFa3TOi04u4XsGVWImIiFoCw0g9nOyVUCoMS7FyqIaIiKhlMIzUQxAEqXeERaxEREQtg2GkAcYwomXPCBERUYtgGGmAq8pQxMqeESIiopbBMNIAF07vJSIialEMIw1QV0/v1bJnhIiIqEUwjDTAs50DACC/uELmlhAREd2eGEYa4OliCCNXSxhGiIiIWgLDSAO8XAz70+QWl8vcEiIiotsTw0gDpJ4RhhEiIqIWwTDSAGPPyFXWjBAREbUIhpEGsGaEiIioZTGMNMCznaFnJL+kAsXlXGuEiIiouTGMNMCjnQPslYbN8rq/uQV/ns2VuUVERES3F4aRBigVAt6dEgUAEEVg++krMreIiIjo9sIw0gjT+gfj7cndAACXrpXK3BoiIqLbC8NIIwW5OwNgGCEiImpuDCONFOjuBAC4dO26zC0hIiK6vTCMNFL76jBSVFaFwlJumkdERNRcGEYaydnBDj6uhmm+X+8+j11nciGKosytIiIiavsYRprg8WEdAQCf70jFrG/isfMMp/kSERHdLIaRJpg2IMjs/rZTOTK1hIiI6PbBMNIEakd7s/sKQZCpJURERLcPhpGb8P3+iziZqZW7GURERG0aw8hNem5VgtxNICIiatMYRproyeEdze6fzi6SqSVERES3B4aRJnp2TGd88bc+cLJXAgBcHe1kbhEREVHbxjDSRI72Skzs4Y/4V0YDMCyCVlJeJXOriIiI2i6GkRvk6mgPV5WhVySrsEzm1hAREbVdDCM3wd/NEQCQVcjN84iIiG4Uw8hN8NcY9qs5n1sic0uIiIjaLoaRmzCooycAYEtytswtISIiarsYRm7CxO7+AIAD569yJ18iIqIbxDByE4I9neGvcYReBA6evyp3c4iIiNokLpJxk8J8XJBVWIYnfjgCAJg+IBix93SXuVVERERtB3tGblInbxez+yvi0yGKokytISIiansYRm6S2soKrNoyLoJGRETUWAwjN+m+vkEQBPNjmQVcd4SIiKixGEZuUrCnM869dyeeMNlALz4tHw8sPYjtp3NkbBkREVHbIIhtoMBBq9VCo9GgsLAQarVa7ubU6YnvD2PryZoAohCA87ETZWwRERGRfBr7/c2ekWYU4OZkdl9/y8c8IiIi+TGMNKNwX/OZNXYKAXomEiIionoxjDSjIZ28zO5X6UV8viMVqVeKZGoRERHRrY9hpBmFeDpbHPt33Bm8/HuSDK0hIiJqG5ocRnbv3o1JkyYhICAAgiBgzZo19Z6/c+dOCIJgccvOvv02lxMEAT88OgBPjuiI/h3cpePxF/JlbBUREdGtrclhpKSkBD179sQXX3zRpOelpKQgKytLuvn4+DT1rduEYeHeWDChK56+Ixw9g9yk493f3ILOr2zC/45nSsdKyquwI+UKKnV6GVpKRER0a2jy3jQTJkzAhAkTmvxGPj4+cHNza/Lz2qrhnb0xLNwL4a9sQpVeRFH1qqxPrzgGtZM90vOv47U1huGbuaPC8I9xXeRsLhERkWxarWakV69e8Pf3x5gxY7B37956zy0vL4dWqzW7tUWCIKDKymyaWd/ES0EEABbtOteazSIiIrqltHgY8ff3x+LFi/Hbb7/ht99+Q1BQEEaOHImjR4/W+ZzY2FhoNBrpFhQU1NLNlJWO03+JiMiG3dQKrIIgYPXq1ZgyZUqTnjdixAgEBwfjhx9+sPp4eXk5ysvLpftarRZBQUG3/Aqs1vx3TxreWX+ywfMufMCVWomI6PbS2BVYm1wz0hwGDBiAPXv21Pm4SqWCSqVqxRa1nIcHd4BeL+K9jafMjruo7FBczt19iYiIZFlnJCEhAf7+/nK8datTKATMHByCsZG+mD2iEwBDEOkRqDE773oFgwkREdmmJveMFBcXIzU1VbqflpaGhIQEeHh4IDg4GAsWLMDly5fx/fffAwA++eQThIaGolu3bigrK8PSpUuxfft2bN26tfk+xS1OZafEkpn9AAD39wuEyl6Jd2sN3Vy+VgqdKGL2D0cwLyYcd/cOlKOpREREra7JYeTw4cMYNWqUdP+5554DAMyaNQvLli1DVlYW0tPTpccrKirw/PPP4/Lly3B2dkaPHj3wxx9/mL2GLenobdi/RqEQzI6P+Xi39POzPx/HpfxSxET6oqt/26qRISIiaqqbKmBtLY0tgGlLnl5xzGwBNGsGhnrg5yejW6lFREREzaux39/cm0YmQsOn4HxeifRzwfUKFrwSEdFtiWFEJkIj0khuUTnKKnUoLK3EHQt34f7F+9EGOrKIiIiaRJapvQSM6+aHtQmZcLJXoqN3O1wtrkC2tszivBd+PYHzucXIL6lAfkkFMgvL0N7NSYYWExERtQyGEZlMiPLD948MQFd/NbxdVTiafg33fLnP4rzadSUnMgoYRoiI6LbCYRqZCIKA4Z294e1qWNytd5AbHhgUjC6+rma7/db20dYUlFboWqmVRERELY89I7cIQRDw7pTu0v3fj17Cc6uOW5x3LrcEcadycFfPgNZsHhERUYthz8gt6p4+gZgfE252zDg8c+nadTmaRERE1CIYRm5hTvZK6ee02DtxVy9Db0hOoWWhKxERUVvFMHILu6dPINyc7TG5VwAEQYCf2hEAkK0tQ0WVXubWERERNQ+GkVuYt6sK8S/H4JNpvQAAvtVhZEtyDrq9sRn7UvMa/VoZ+deRkd+44R1RFFFWySJZIiJqHQwjtzgHOwWE6hXS/DSO0vFKnYg1CZexNuEy3lyXjOUHDfsBXSupwNbkbFTq9EjJLsLGxCyUVepw1+d7MOLDHSi8XgnAsEvwFSvrmgDA3OXHMOC9P5BbVN7Cn46IiIizadoU4zCN0arDl7Dq8CXp/ogu3pj8+V7kFZfjncnd8NraZADAggkRuFYdQtYdv4wHozvgyR+O4NCFfGx8ZhiyC8ugVAgY2NETALAhMQsA8MuRDPx9ZFhrfDQiIrJh7BlpQ7xdVYjwc63z8U/iziCv2NCbYQwiAPDDgYvSzysPZaC4vAp/ns1DWaUeH25Jwd+WHsS0JQdQeL3SbHimuIx74RARUctjz0gbolQI2PjMMOSVlGPAe9ssHv/lyCUrzwIuXSuVfk7O1OInk3CyKSlb+vnexfswJtJXum8MNkaVOj3slcyvRETUvPjN0sYoFAJ8XB3RxdcVDkoFNjwzFEPDvMzOCfF0rvc1Yjedtno89UoxFu08J92/kHcdJzO1OJtThMsFpejzThw6v7oJ//z1BCp1nM1DRETNgz0jbdTKJwahpKIKge7OZtN8Nz4zDJ182qG8Sg+FIOD+xftxMksLjZM9ZkWH4D/bUxv9HkfSr+GeRXtRqRMR6a9GUfWwzc+HM9A/1AP39Q1s9s9FRES2hz0jbZR7OwcEuht6QCID1AAAhWD4WWWnhNrRHi4qOyx/fCBendgVa+cMwSSTJeTv6xuIj6f1BAD0C3HHqbfH45/jI8zeQ6cXUVaph04vIvFyodljZ68UST//cOAiPtt2FqIotshnJSKi2xt7Rm4Dz4/tDJWdAn8dEGzxmJuzAx4b1hEAoNeLCPZwxpWiMswdFYYQT2f0DfZAe3cnKBUCYrr64P82Wx/CqS2zwDAtuKS8Cq+tSQJgqD9ZOqsfAmrtKlxepYODsmaKMhERkSn2jNwGXB3tseDOrgj1alfveQqFgF9nR2Pr/BHo4NUOgiAg2NMZSoUhJIT7uuKTab0Q4eeKLr41s3Yi/dUWr3X04jWUVuhwOrumh+RklhbP/pyA9KvXse54JkRRREb+dfR+Ow5vrEu2eA0iIiIAEMQ20Leu1Wqh0WhQWFgItdryi5Ga375zeXjyhyO4v18QHhkaiiEfbLc4Z2Z0CDr7uuLV6p6R2pY/NhD/O5GJFfEZAIALH0xs0TYTEdGtpbHf3xymIasGd/LCiTfGSkMrn03vjadXHDM758cDF9Ghnt6Y5Ewtym+BPXRKK3SwVwqw47RkIqJbEn87U51Mazwm9QyQhmuGd/bGkDBP6EXgfG5Jnc9/f9Mp/H70crO1RxRF7DuXh8MX8vGXz/7Esr1pDT7nekUVhv1rO6Z+tb/Z2kFERM2LPSPUaF/P6oef49PxYHQH/HrkEvamXgUAPDAoGO9O6Y595/JwNqcYjvYK/PO3RNQeACyr1MHRXtng+4iiCL1o+NO0N2Pd8UzMW5kg3U+6fBIPDQmt97VOZxchr7gCecUVuKItg0+tJfWJiEh+7BmhRmvv5oTnxnaBt6sKA0LdpeP39wsCYBjamTW4Azp6u1h9fkH1/jhGX+5MxQdWFmD75I+z6PTyRoS9sgnvrj8pHd9z1nKX4rM5RVh1KEPaALC0QofrFTXL2JsuaZ+QUdCIT0lERK2NPSN0Q3oEumFQRw84O9ihe3uN2WPhPnWEkdIK+GkccTT9GnR6Ef/anAIAOHIxH3nFFXhnchSGhnvh021npecs3ZOGLn6ucFHZYU2C5ZDPmI93AwB2nc3Fx/f3wp3/+ROVOj02zx8OAUB+SYV07hM/HMGef46S1mepiyiKnIZMRNSKOJuGWsTR9Gs4d6UYL/x6Qjr28xOD4Kt2xMiPdlp9jmc7B+x+cRS6vbHlht5z+eMD8bevD0r31Y52uCPCB2sSMqVjr/8lEo8MDYVeL+Ji/nUEezgjPi0fHbyc8d2+i+jfwR2vrknC+Cg/vDGp2w21g4iIDBr7/c0wQi2mrFKHiNc2mx2bMTAYPx1Mr/M5ARpHZBaWWX3MyV6JUpNdhWv728BgLK/ntQHgr/2D8OZd3fDIskPYd+5qvec2x1TkayUVSMgowMgu3tCLwCurE9GtvQYPDgq56dcmIrrVNfb7mzUj1GIc7ZVYNKOP2bG6gojKzvBXsa4gMjbSF6feGY/fnhosHas9PFRfEInwMyzilpJThF+PXGowiAAw2/PnRr2z/iQeXnYI/447g7iT2Vh5KENasbY53aobF36xIxVf7z4vdzOI6BbHMEItakJ3f9grG66/eGhwh3of91GrAAD+mprZMHd297c4787uflafH93JEwBwLL3AbJE2v3pm13R9fTNyi8qtPlZSXmU1APxyOANvrE2CTi/iaPo1/H7MUOfy2fZUpOdfl84rr6q7h8dobcJl/LD/Qr3nVOr0WJtwGd1e34Lfj15q8DVbU25ROT7ckoL3Np5CSXlVw08gIpvFMEIt7vW/RAIA2jkYpvX2DNTAzdne7Jz7+wdJP/99ZCckvzUO82PCpWM+robQ4O2qko49GB2C2Hu6m73OAwND8MzocNTWL8QD7U32zGnnoMTxN8Zi/4I7sOuFkfjXfT0sQpNOL+J/xzNrvxQKrlcgOnYbZiw11KdcKSrDpsQsJF0uxAu/nsB3+y/imz1puOfLfWbPS7yslX7OK64we+y7fRfw3M8JKKvUQRRF7E3Nw7yVCXhtbTLGfbwb53KLAQC/H72EP07mSM97ZNkhzFuZgAqdHs+tOm7R1rpcvFqCo+nXGn1+bTtTruCjLSnQ6+se5S24XvMZ84qthzoiIoCzaagVPDAoBJN6BsDRXokNJ7IwppsvXFV2eOt/J7Fs3wUAQIiHM/4+shM2JmZh1uAOaKeyQ68gN+k1fKpDiL1Sge3Pj0CVXoSLyg7TBwTDy0WFx78/DLWjHfp2cMegjp6YPiAIcSdz8PraZAgCEOHviq3PDsf7G09h3fFMfDS1JzROhkAU4tkOIZ7t8NWuczhXaxE34749AHA6W4visiqczyuBtqwK8Wn5uJBXgr98tgfFtf7l/97GUxbXIe5ktvRzblE5EtILsOTP83hqRCdp754OXu0Q5uOCv/90VDo3JacI9y3ah9+eGiwFjufGdEbB9Ur8aWW6c0NEUcSID3cCQKNmF9Wm04t46NtDAIBOPu1wd+9Aq+ddLTEPIyGe9e+dRES2i2GEWpwgCHBzdgAA3Nu35ourwmSYw06pwIvjI/Di+AjpWJjJFGGPdg7Sz7XXMRkT6YvvHxkAd2cHqOwMvS/+Gic8OCgEvYLcYK9UoFP1c967uzvenhxlFjKMAt2dLcKIcW2U87nFmPz5XlTpRUwfUNOL8+XOVIsgUpeyyprPm1tUjjnLDYFj9o9HpOOrDmegSmfZ23DteiX+8tke6f6/485YnOOiatz/zlkmdTnnc0uaFEZyi8ox/pPd0v1TWUXwaJcLhWB4/4LSSozq4gPAfFp1blGFxWu1hMyCUjjYKeDlomr4ZCK6ZTCMkGzu6OKD5QfT4V5ryMYoQFMzrOLiWP9f1eGdvS2OCYKAHoFuFsetBREAGB/lh11ncs2O5RQZvrg/35Eq7bPz44GaQtlVh2+sTuPx7w9bPX7pWmmdz7leUX+dSXF5FaYvOYAVTwyq97zEy4XSzzO/iceiGX1gp1RgQAcPaOr4b2G0Mj7drMfj8rVSzPom3uycXS+MRIhnO7PzcovLUVJeBWcHZYut4aItq8TgD7ajnYMSSW+N41oxRG0IwwjJZnRXHyx7uD+6+luf7qVQCPjgnu44nV2EQaGeLd6e6QOCcUeEDwa+v006tvxgOval5pn9K78+PQI1OHGpsOETW8j+81dxraQC7iY9SbUlXTZv31PVQ0IxXX3w0oSuCHBzhLNDza8GbVklzueWQBRFLKzVI7MxKcvi9VOyi1BaqTObNfTL4Qy8u/4kyqv06BagxoonBkHtWH/waapTmYaanJIKHdYmZEKhEDC+mx8c7CxL4/al5uFfW1Lw0dSeZj1wRCQPFrCSbARBwMguPvCtZ0bLXwcE4827ukFRR29Gc/NVO+Lbh/qbHbtw9Tq01cvKR7W3DE59Q9yx5MG+eHNSJH5/ajBmRjf/GiJn3p2AeVYKc4GaactG5/OK632tujY3/OPUFcT8exf+8YuhLqWsUoerxeV4ftVxTPliL+6uVZALwGL/IQBIzS3G2/87aXbsxKVCqWcpOVOLUR/uxNmconrb2VSms5Xm/5yAZ1Ycw7M/J5ids+54Ju5dtA9/W3oQCRkFeGNdw9OsT1wqQHTsNqyMr38Nm5ZQX4Ew0e2EYYSollERPlj/9FCL4072SqybMxRJb43DgFAP6fjkXgEY280PDw0JhZ1SgbcnR2H3C6OwYIKh/iXYwxnvTO4Gz3YO+GhqTwR71NRohPm4SAugvXxnBI6/PhZ+akd08XXF9AHB0nkOdgo8O6YzXhjXxaJdprOOAFjUvdSWrbW+lovRxsRsaMsq8fSKY+j77h+IM5m90xipOcVoaITkakkFJn2+B3cs3InHvjtcb8+TKIoWs3FEUURprWGr83mWn7v2fkSf/HEGRy7WzCIyfY3i8irM+iYeHV7agPGf7EZidQ/XMyuOIauwDC/9noj4tPz6P1gzOpWlRfc3t+Dz7WctHsvIv47xn+zGL4czGvVabWBtS7JxHKYhsiLCzxUxXX2Rnl+CMzmGnoYwHxcoFAJcVHaYEOWH+LR8CAJwbx/L2STBns54dGgo/DSOGBjqCT+NIx6M7gAAmNIrAIWlldiQmIWRnX3gq1Fhcq8A9Apyg51SgbjnhsNOocDm5CysqPWvceN6KQDw9B1hmDEwBL5q82JN4zRgU0mXC+HsoERHbxfkNBBGAGDG1wfNakvqEu7jglCvdnhubGe8vjYZ8Wn52H02DxVW1lGZMTAYfYLd8bzU86LH+dwSnM8twfoTmZhZfX1q+++eNLy74RQ+vK8HplZvyvjR1hQs2X0eSx7sh1ERPtCWVSIl27Kn5XJBKR7870H8eTYPge5OFjU5xoJnAHhzXbJUM3Q6uwh/+/oAtv1jBC5crelxuf+r/Uh6a1yji4VvxrM/J6CkQoePtp7B3DvMA+eC3xNxOrsIL/x6AlP7BWFFfDqOpV/D25OjLHbG/scvx3Ew7SrWPz1MmkF2M/48m4uP487g/Xu6I8Kvpqcw7mQOdHo9xkdZrv9D1BD2jBBZYadUYOmsfvj970PgXL0+yiyThdmm9Q/Cu1OicPDl0WhXxxeTnVKByb3aw0/jaHHc00WFmdEdEOzpDJWdEv06eMBOafjf0dXRHk4OSkzu2R7v390dcc8Ol57bJ9gdM6NDoFQIGBPpCz+NIwRBwIZnhkobFH616zzGf7JbWpTtdLYWd32+B3cs3IUOL22ot0jWyFoQGdzJE54mtSh3926PuOdGYMnMfojwU+OD6jVf8orLpWEtB6UC706JwqCOHnhyeCcMC/eSVts1VdficttO5eDdDYZp0sZ9jiqq9PjxQDoqdSJeW5uEU1laDPlgO7afvmL1NYzTn6197qTLhVh+MB1Jlwux5pj5RoxF5VWI3Wi5q/TFq/X3PDXW+dxis4XzEi8V4lp1D1FyZiFOm4Qr43tmFZbihV+OY09qzZTutLwSLPg9EasOX5KmyhtV6vT49cglZOSXmq1PczMe/G88jqYXYN6KBOlYblE5Hv/+MGb/eNRsfRnAMBW8sT0zp7O1uCrDmjSnsrS4b9E+7DuXh6KySuxNzWt0m6t0esxdfhTvrD/Z8MkyaMwCi7cC7k1D1IDT2VoIENClVm2GXERRRFmlHk4O5v8C/vNsLh78b83MlvVPD0VUew2e+P4wtjbwRaSyU2BAqAcWTu2J/+5Nw1e7apZwnx8TjnmjwyEIAqp0epSU67DtdA7GRPrC1aQItUqnR9grm8xed+uzw9HZ1/y6XSkqw5yfjuLQhZrhkukDgi0WsCuv0qHLq+Z7G/Xv4A7PdipsTs7GjVIIQNxzIzB64a5GP2f2iE7YkJiJjPxSfDytJ0Z09jGbbm7N2ZwiPPnjEcwe3gmJlwtxZ3d/uDnb4/82n4bKToEtyTmYOyoM/xjXBQkZBZjyxV509GqH7f8YifGf7DYLIwBw6JUYzFt5rN6tDLq31+B/Tw/F9YoqnM4ugspOgYn/MUwJf+0vkXh0aGijPzMAHEu/hvc3nsKL4yPQLUANJ3slQhdsBGAYOjzz7gQAhinpL1aHRePfO+Pzpy7ej5cmROCxYR3rfa99qXmY8d+DaO/mhM+m90avIDdoy6qapTenPm+uS5ZCnKO9AqMjfLEhMQtvTIrEw0Pqvl5VOj1eXp2I6xU6rD9hKOROfmtcnf84kcPn28/ikz/O4uNpvTCpZ4AsbWjs9/etc9WIblGmXdG3AkEQLIIIAGktFSPTdUlq82zngE//2hs/HbyI5EwtVj0ZLfXgLJjQFVkFZVh3PBNPjuiI+TGdpefZKRXQOCtwj5WhKWPPjilrxck+ro547S+RuOvzvdIxazUhplOojUwDTJiPC1Kv1F+sa41ehNlqvKaeH9MZd3T1gZO9EjOWHpTWZBkT6YPLBaXIyC/Fsz8fh4vKDjv+MdJsReD4tHws2pmKd6ZEIdDdGf/45TjO55bgxd8MX9I/HLgIZwel2RTtxbvO4dK16ziZZZgJdD6vBFe0ZRZBBABmLD0gDRnWJfFyISJf34wwHxeLWV0XrNTU6PQi3lyXjJ5BbrjPZA0gnV5EZkGpVLQ8dfF+2CkEPG0yXKQzKa41nRJ/6VopotprEJ+Wj/u/2g8AeHfDKTw6NBSCICAj/zq+3JmKJ4Z3QqhXO1Tp9PhmbxoWbj0DUTQ8/+4v92FAqAeOXLyGtXOGSOGmsa6VVOC5VQn464BgjOtmfYsIwNBzZNqbVFapx4ZEQ7CI3Xi63jCyJzXPYmr/V7vPY8bAYHi2c7D6/0NrKq0e4gOAp1ccw8gu3nB1tMe/Np/G6mOXserJaAR5NG3Bw5bEYRqi20R9++xM6RWAX2ZHS/d91Y4YGu6FRQ/0xe4XR1kMJf3rvh74z/Tedc7gaSx1HevD9Ah0w4rHB+HJ4YZ/LZt2zadeKcasbw812O39/t3d8erErgj1avrKrqZ1FaZTe7sHatAtQIOO3i7YPG84PryvB354dAD6hnggyL0mwBSXV1kU9t7/1X7sSMnFgt8TAQA5WsvhhtprxVTpRaxJyDQLGQOqp5bXnrnVUBAxfQ9r08utzbLafvoKfjhwUZpBBRhmUd2zaB+G/WuHRVs//qNmardOL2LxrnPYm5qHcyahMLOgFKeztVIQMTpe3aZP/jiLFfEZGPXRTuj1In47egnvbzwtzbYyik/Lh04v4tNtlgW8Dfl23wXsSMnFkz8cQZXJUFhaXgl+P3pJGoI5W881rdDpIYqGIaYTlwpQVFZp9nillcUJ/7PtLAa+v02qi5KDMSSuO24+7Hj2SjEOnr+KL3eeQ1ZhGdYmXLb2dNkwjBDdJuqb/jy6q6/UzQ6gwS5bR3sl7uoZYLbeSGMsfqAvHJQKtHdzwlt3dat34bHoTp4YE+kLwPAl8eOBiyit0GHu8qPYXWvxudr6hbijX4g7HhvWETv+MRKb5g1DzyBDwKntLz38kRZ7J967OwoAMKt66vWvs6PxybRe0t5JAMwWydM422NqvyAMCzcsqOdZa1XXj7am4NK166io0uM5kynEx6tn8NQ3o8jJ3rJnq7aRnX0aPMfovr6BmNjDsnC0T7Cb9PPJTK3FTtT5JTWByTiz6I9TOdJnaMgHm05jxtKDZj05lwtKsXCr5QrBU77Yi0Hvb8NvJhs6HrqQjw2JhiE3hQB8NLWnxfOqbmBHam1pTXD486yh/mPpn+cx6qOdeG7Vceys/vuVnFl/kXaOthzvbTiFuz7fi8Gx26XandQrRXh+VUKdz1ubYLmnFWCYJj5v5TFcunbd6uPbTuXgmz1p9bbJ1NmcIqw+dgnPrzqOtLwSPPXjEfR9Nw6HL+Tj+/0Xzc6958t9eHjZIem+cabZyUwt/v7TkRvqZWxOHKYhuo28MK4LPtySYnG8V5AbnB3s8ONjA1BaocfQcK8Wef/xUX44896ERp9v/IK/dr0Sr65JwnsbTqG0su6Cu0B3J2x4epjFSrFd/dVYO2eI2bEBHTzw5l3d0NXfFYIgYMbAEHRvr5GGs/p18EC/DoaC2OGdveGnVtVbBzK5VwDWHc9E/xB3LN2ThvySCizbewF2SoW0OzNg6Jn47cgls2X3a1v/zFC8vjYJe1Prrv8YFu6Fz3ek1vm4kfFLvLi8Cl39XLEnNQ9+ake8f093ODvYoVKnx7D/24FsbRk2JGaa7SVk2lPT9fXNiPBztTpE1BTxaflIquNLvva08vUnsrCvuhg37rkR6OTtAr1elIa2AOBgWj7+b/NpzB7eqcEVgo1MZ4ztSLmCFfHpZnVT/0vIRPLlQqRU94x09GpndWr4Q9/GS9ejqLwK209fwcNDQnH/VwekIu26aMsqLRb2Mw5N6vQiPv9bH7PHRFHEo98ZVmbu18EdPQLdUF6lw5pjl6FUKHBP7/Zm/+D4fv8FvL42WbpvGvL+uuQAqvQilAoBgzt5SkXcpv+9D124hud+TpD+7p7OLsL250fW+5laEsMI0W1kzqgwPDWiEwTB8Ev4kWWGX26B1UMMfUM86nt6q/N0Mf/yNw0ijwwJRXJmIQ6arO3x6sTIBr+QPNs54GpJBUZF+CAywHyow9r2AA52Cnz/yIAG2+rlopICj1Ip4Ktd57HUyr9iq/SiWTd9pL9aqgkxCtA4IfbuHth1NheOdgppppCpPiHuDbYJANTVBZ4uKjvMvSPcYhqwvVKB+/oG4vMdqdhz9iru7h2IKp0e/9l21qIY1jSIxHT1wYxBIXj420NoisZMCTf64YDhX+/Dwr2kkBjua177dL1Ch0U7zyG7sAwfT+slHf/tyCUIgqHeKLuwHI8PD4WbkwNKKqqQYdLzULuHAIBZeASAd++Owt++PmhxXu1gduTiNRxNL7BYF+ez6b3x9IpjZsdSrxSjT3DNf0PTGpvswjKIooglu8/DR63C3b0DkWsyVLnmWCa6+qvxxtpkrDxkWEtm1eEMPDIkFOOjDDUwX/9ZU2ReW1X1e4V5uyAyQG22oaabsz3sFALyiivMrkNdiyG2FoYRotuM8V9Po7r44O3J3dDF1/WW3afFVWVnUdQJAHHPDke4ryteXZMohZHZIzphbPWwTn3Wzh2CP8/m4Z4+7VukzQDQO8gyKBx/YyzKKnX4v82nsTU5Bz0CNXh9UiQi/NTo8NIGs3OdHJQI9nTGg56GISN3ZwesO56JQxfyMbKLN6b0ag97pWGGU3xaPl4c3wX/2mzo8TLWyRj/Fe3j2vCmgMa6mj2pubiQV4LT2UX4z/b6e13uiPDFqC4++EsPf2xOysbM6A5YHn8RZZV6vDQhAg8OCoGdUjCb8TQkzFPq7ekZ5CYN94zq4g03Zwc42iuwIt58oTaPdg54865u0v2o9hoM7+xtMVS3KSlLCiPXSios6jLWn8hEeZUehaXmtR2mrK0109G7HaI7etbZqygIwLcP9cdD3x6SZs2YempkJ0zqGYDh4d7o+fZW6fjZnCKzMHLKJJD6ahyx/9xVxG4yTB0P93E1Kzj/Zm8a9KJoFuzi0/IRn5aPjt7t8MqdXZGR3/AU/cgANdyczAN/vxB3DAnzwlv/u7WmIjOMEN2mBEGocyGxW4UgCOjs62q2UupTIzshvHo68IODOuDHA+kYHeGDlyZE1PEq5gLdnc1Wr20J/hrLYmGNkz00Tvb49/29LB7rFqBGcvXeOWOsBKqYSF/EWDm+5MG+OHThGkZ18UaP9m44mVUozUp5665u+GhrCj68z7LOojbjrKYcbTlGfrSzUdcywM3wnI+m9sTrkyLh4+qI1ydFWpz3wKBg/HggHZN7BeAfY7tg7Me7UVqpw7192kth5OuZ/WCnVCA+LR8r4jOgEAyzr/R6EVvmDzeblWSvNPRU6fUihv1rBy4XGL50yyr1+GhLCh4e0sHqENiVOtaqMTU20g/f7K3pzRrU0QNv3RUFQRAwZ1QYVsSnW4QVf7Uj+nfwgEIwzMSqza26Z0rjbI95o8Olgtsdp3MxrX/N38MD52t6obSllVh3vKauxNrMt2X7Lljdffp8bokURK3xUztKw2GR/mqLmXcdPNvhgUEhWBGfblEUnVdcLtuO1wwjRCSrCVF+Uhhxd7Y329uni58rDr0SU+fOznKpHUZeHG+5TL+p/87qj22nczCqi0+jejKM3JwdpPAyNNzLrNZn1uAOeHBQSKP2baq9Sm/tBd4Aw3oyvYLc8FD1sIx/9a7ZjvZKi1VdTb11VxT6d/DA4E5e8HZV4ZfZ0cgtKsewcC9sTc5BiKezNM11QKgHfnsqGn4aJxRer4Sro51ZEDGlUAjw0zhKYQQw7J699WQ2imvVazw0uIM0RddeKSC6kxfCvF3MggcAjOvmCwc7BcqrdJg3Ohxuzua9BtZW1g3ycEY7lR0i/CyH2wDDsIfRs2M6487u/hj3yW5sTs7G/V/tx9uTuyHCT40D52uGG68WV+BkpuVr1VZ7yntt9krBbFaPIABd/V2lMNIr2A1RARrsTMnFH6cMNTOdfFxgr1Tg5yeiMWrhThRcr+lJOpNTxDBCRLbp0aGhKKvUo38Hd/QJcbf44qvry0pOpjNrpvULwlMjOtV7vp/GETMGNv8Gio3dQNLH1Tw8mdZCONkr8dPjA9Er0A2CAHT2dUFJuQ4dvBq3BoVSIWByr5ohMdM1QX58bKDF+ca6pbrWejHlYGWtjtr/mn94SAfc3y9ICiMvTeiKR4eG4npFlRRGega54bGhoRjY0RMDO9a9A/gbk7ph+tcH4GivQFmlYRaPcS+p9u5OVsOIi8o8KHfxc5UKYuPT8nHvl/vw+qRIKQwAsPo6dbFTCHjv7ij887dEs+NT+wbilYldseD3RGndn47e7cw2h+xdvcXE0ln9sObYZRy+mC+tJ+PezgHP3BGOt6un0P/rvh4I85ZvB2uGESKSlZ1SgXkxN7eeSWtTmoSAYE/nW7Ymx0jtZP1X/Wt/icRdPQPMAt/6p4dBL4pm+/bIxU5Z/3W9v18gXp0YCYUA9A52Q3KmFjFdDVOinR3sMKijBy4XlOLbh/o3uGIuYJhufvS1Mdh/7irmLD8KAFIR9AODQhB3Mgf39Q3E/93bA51eNqxEW6W3nHoc7usizc4pqdBZBAmjAR08EH/B0GMyvpsfAt2drBZFT+sfjPv7BWF5fDpeWW3YaXpouBfcnB2w6IG+ZueOj/LDiUuFiPBzNVt4bUrv9pjS27yO6m8Dg5F0uRAjI3xwl0wrtBoxjBAR3YDnxnTGluRsPNACPR7Nra6w1NGrnUXPk4OVvYPkEuLpjD/rWfPM0V4pBcPvHhkAbWklAt1renRWPD4IVXoR9k1YDdWjnQOiO3kiqr0a3du7SfVHIzp7Y9vzIxDs4WwWRsN9LLeJCPNxwZZkyy0Y/nxxlNlicpN6BWB4Zy98u/cCnh/bGV4uKqw7nolRXXzwc/WOzMaZMcbp6V18XXHiUmGd4eHxYR3h4+oohbL6ONor8W+TGUpyYhghIroBz4wOxzM3uUJtaxob6YutJ3Pw5IiO0t5Dt+IQmKlnYzojs6AM9/cLhLasCp28XdDB0xl93/0DABAVUDMkpHa0t1jXQxAE2DfQu2KNRzsHrH96mMVx0y0Xtj47HJevlVpMHwcMs6OsCXQ3H5q6r08gnByUZtOxDywYDYVCwPm8Yhy6cM1iFWPD+jh1T9E3TuVuaxhGiIhswOd/64O84nK4ONpJYSSgEXUbcvJ0UeGbh/pbHI97djj2pubh7hacvt2Qzr6uFptAGhkLjTVO9vjpsYF4ZuUxzBkZBkEQ0L+DO5Iua7HogT5W95gy1gEteqAv3t946pafEddcuGsvEZGNiU/LR2mlDiM6e8vdlNvW0fRr8Nc4SrOSjCqq9Cit1LX4bsS3Cu7aS0REVg0IvbVW4r0dmS54ZsrBTnFL1eXcKnhFiIiISFYMI0RERCQrhhEiIiKSVZPDyO7duzFp0iQEBARAEASsWbOmwefs3LkTffr0gUqlQlhYGJYtW3YDTSUiIqLbUZPDSElJCXr27IkvvviiUeenpaVh4sSJGDVqFBISEjB//nw89thj2LJlS5MbS0RERLefJs+mmTBhAiZMmNDo8xcvXozQ0FAsXLgQANC1a1fs2bMHH3/8McaNG9fUtyciIqLbTIvXjOzfvx8xMTFmx8aNG4f9+/fX+Zzy8nJotVqzGxEREd2eWjyMZGdnw9fX1+yYr68vtFotSktLrT4nNjYWGo1GugUFBbV0M4mIiEgmt+RsmgULFqCwsFC6ZWRkyN0kIiIiaiEtvgKrn58fcnLMdy/MycmBWq2Gk5P1fRFUKhVUqlt7AyciIiJqHi3eMxIdHY1t27aZHYuLi0N0dHRLvzURERG1AU0OI8XFxUhISEBCQgIAw9TdhIQEpKenAzAMscycOVM6f/bs2Th//jxefPFFnD59Gl9++SVWrVqFZ599tnk+AREREbVpTQ4jhw8fRu/evdG7d28AwHPPPYfevXvj9ddfBwBkZWVJwQQAQkNDsWHDBsTFxaFnz55YuHAhli5dymm9REREBAAQRFEU5W5EQwoLC+Hm5oaMjIx6tyAmIiKiW4dWq0VQUBAKCgqg0WjqPK/FC1ibQ1FREQBwii8REVEbVFRUVG8YaRM9I3q9HpmZmXB1dYUgCM32usbExh4Xc7wu1vG6WMfrYh2vi3W8LtbdrtdFFEUUFRUhICAACkXdlSFtomdEoVAgMDCwxV5frVbfVv/xmwuvi3W8LtbxuljH62Idr4t1t+N1qa9HxOiWXPSMiIiIbAfDCBEREcnKpsOISqXCG2+8wdVea+F1sY7XxTpeF+t4XazjdbHO1q9LmyhgJSIiotuXTfeMEBERkfwYRoiIiEhWDCNEREQkK4YRIiIikpVNh5EvvvgCHTp0gKOjIwYOHIj4+Hi5m9Sidu/ejUmTJiEgIACCIGDNmjVmj4uiiNdffx3+/v5wcnJCTEwMzp49a3ZOfn4+ZsyYAbVaDTc3Nzz66KMoLi5uxU/RvGJjY9G/f3+4urrCx8cHU6ZMQUpKitk5ZWVlmDNnDjw9PeHi4oJ7770XOTk5Zuekp6dj4sSJcHZ2ho+PD1544QVUVVW15kdpVosWLUKPHj2kBZiio6OxadMm6XFbvCa1ffDBBxAEAfPnz5eO2ep1efPNNyEIgtktIiJCetxWrwsAXL58GQ888AA8PT3h5OSE7t274/Dhw9Ljtvh71yrRRq1cuVJ0cHAQv/nmGzE5OVl8/PHHRTc3NzEnJ0fuprWYjRs3iq+88or4+++/iwDE1atXmz3+wQcfiBqNRlyzZo14/Phx8a677hJDQ0PF0tJS6Zzx48eLPXv2FA8cOCD++eefYlhYmDh9+vRW/iTNZ9y4ceK3334rJiUliQkJCeKdd94pBgcHi8XFxdI5s2fPFoOCgsRt27aJhw8fFgcNGiQOHjxYeryqqkqMiooSY2JixGPHjokbN24Uvby8xAULFsjxkZrFunXrxA0bNohnzpwRU1JSxJdfflm0t7cXk5KSRFG0zWtiKj4+XuzQoYPYo0cPcd68edJxW70ub7zxhtitWzcxKytLuuXm5kqP2+p1yc/PF0NCQsSHHnpIPHjwoHj+/Hlxy5YtYmpqqnSOLf7etcZmw8iAAQPEOXPmSPd1Op0YEBAgxsbGytiq1lM7jOj1etHPz0/88MMPpWMFBQWiSqUSV6xYIYqiKJ48eVIEIB46dEg6Z9OmTaIgCOLly5dbre0t6cqVKyIAcdeuXaIoGq6Bvb29+Msvv0jnnDp1SgQg7t+/XxRFQ8hTKBRidna2dM6iRYtEtVotlpeXt+4HaEHu7u7i0qVLbf6aFBUVieHh4WJcXJw4YsQIKYzY8nV54403xJ49e1p9zJavyz//+U9x6NChdT7O37s1bHKYpqKiAkeOHEFMTIx0TKFQICYmBvv375exZfJJS0tDdna22TXRaDQYOHCgdE32798PNzc39OvXTzonJiYGCoUCBw8ebPU2t4TCwkIAgIeHBwDgyJEjqKysNLsuERERCA4ONrsu3bt3h6+vr3TOuHHjoNVqkZyc3Iqtbxk6nQ4rV65ESUkJoqOjbf6azJkzBxMnTjT7/AD/rpw9exYBAQHo2LEjZsyYgfT0dAC2fV3WrVuHfv36YerUqfDx8UHv3r3x9ddfS4/z924NmwwjeXl50Ol0Zn/xAcDX1xfZ2dkytUpexs9d3zXJzs6Gj4+P2eN2dnbw8PC4La6bXq/H/PnzMWTIEERFRQEwfGYHBwe4ubmZnVv7uli7bsbH2qrExES4uLhApVJh9uzZWL16NSIjI236mqxcuRJHjx5FbGysxWO2fF0GDhyIZcuWYfPmzVi0aBHS0tIwbNgwFBUV2fR1OX/+PBYtWoTw8HBs2bIFTz31FJ555hl89913APh711Sb2LWXqDXMmTMHSUlJ2LNnj9xNuSV06dIFCQkJKCwsxK+//opZs2Zh165dcjdLNhkZGZg3bx7i4uLg6Ogod3NuKRMmTJB+7tGjBwYOHIiQkBCsWrUKTk5OMrZMXnq9Hv369cP7778PAOjduzeSkpKwePFizJo1S+bW3VpssmfEy8sLSqXSopo7JycHfn5+MrVKXsbPXd818fPzw5UrV8wer6qqQn5+fpu/bnPnzsX69euxY8cOBAYGSsf9/PxQUVGBgoICs/NrXxdr1834WFvl4OCAsLAw9O3bF7GxsejZsyc+/fRTm70mR44cwZUrV9CnTx/Y2dnBzs4Ou3btwn/+8x/Y2dnB19fXJq+LNW5ubujcuTNSU1Nt9u8LAPj7+yMyMtLsWNeuXaUhLFv/vWvKJsOIg4MD+vbti23btknH9Ho9tm3bhujoaBlbJp/Q0FD4+fmZXROtVouDBw9K1yQ6OhoFBQU4cuSIdM727duh1+sxcODAVm9zcxBFEXPnzsXq1auxfft2hIaGmj3et29f2Nvbm12XlJQUpKenm12XxMREs18YcXFxUKvVFr+I2jK9Xo/y8nKbvSajR49GYmIiEhISpFu/fv0wY8YM6WdbvC7WFBcX49y5c/D397fZvy8AMGTIEIulAs6cOYOQkBAAtvt71yq5K2jlsnLlSlGlUonLli0TT548KT7xxBOim5ubWTX37aaoqEg8duyYeOzYMRGA+O9//1s8duyYePHiRVEUDVPM3NzcxLVr14onTpwQJ0+ebHWKWe/evcWDBw+Ke/bsEcPDw9v0FLOnnnpK1Gg04s6dO82mJV6/fl06Z/bs2WJwcLC4fft28fDhw2J0dLQYHR0tPW6cljh27FgxISFB3Lx5s+jt7d2mpyW+9NJL4q5du8S0tDTxxIkT4ksvvSQKgiBu3bpVFEXbvCbWmM6mEUXbvS7PP/+8uHPnTjEtLU3cu3evGBMTI3p5eYlXrlwRRdF2r0t8fLxoZ2cnvvfee+LZs2fFn376SXR2dhZ//PFH6Rxb/L1rjc2GEVEUxc8++0wMDg4WHRwcxAEDBogHDhyQu0ktaseOHSIAi9usWbNEUTRMM3vttddEX19fUaVSiaNHjxZTUlLMXuPq1avi9OnTRRcXF1GtVosPP/ywWFRUJMOnaR7WrgcA8dtvv5XOKS0tFf/+97+L7u7uorOzs3j33XeLWVlZZq9z4cIFccKECaKTk5Po5eUlPv/882JlZWUrf5rm88gjj4ghISGig4OD6O3tLY4ePVoKIqJom9fEmtphxFavy7Rp00R/f3/RwcFBbN++vTht2jSztTRs9bqIoij+73//E6OiokSVSiVGRESIS5YsMXvcFn/vWiOIoijK0ydDREREZKM1I0RERHTrYBghIiIiWTGMEBERkawYRoiIiEhWDCNEREQkK4YRIiIikhXDCBEREcmKYYSIiIhkxTBCREREsmIYISIiIlkxjBAREZGsGEaIiIhIVv8Pg6/6YbJ5Ik8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(torch.tensor(l).view(-1, 10).mean(dim=-1))"
   ]
  },
  {
   "cell_type": "code",
   "id": "5e9dabdfa0f06539",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-08-23T14:55:47.229233Z",
     "start_time": "2025-08-23T14:55:47.217343Z"
    }
   },
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "vocab_size = 100\n",
    "embedding_dim = 30\n",
    "\n",
    "# 创建一个 Embedding 层\n",
    "emb = nn.Embedding(vocab_size, embedding_dim)\n",
    "\n",
    "# 查看其权重\n",
    "weight = emb.weight.data  # 形状为 (100, 30)\n",
    "print(f\"Weight shape: {weight.shape}\")\n",
    "# 计算均值和标准差\n",
    "mean = weight.mean().item()\n",
    "std = weight.std().item()\n",
    "\n",
    "print(f\"Mean of embeddings: {mean:.6f}\")\n",
    "print(f\"Std of embeddings: {std:.6f}\")\n",
    "\n",
    "# 计算理论上的标准差\n",
    "theoretical_std = 1.0 / (vocab_size ** 0.5)\n",
    "print(f\"Theoretical std (1/sqrt(vocab_size)): {theoretical_std:.6f}\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weight shape: torch.Size([100, 30])\n",
      "Mean of embeddings: -0.002624\n",
      "Std of embeddings: 1.008776\n",
      "Theoretical std (1/sqrt(vocab_size)): 0.100000\n"
     ]
    }
   ],
   "execution_count": 33
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
